Getting Kafka Connect JMX metrics reporting into Datadog
Firstly, your Datadog agents need to have Java/JMX integration.
Secondly, use Datadog JMX integration with auto-discovery, where kafka-connect
must match the container name.
annotations: ad.datadoghq.com/kafka-connect.check_names: '["jmx"]' ad.datadoghq.com/kafka-connect.init_configs: '[{}]' ad.datadoghq.com/kafka-connect.instances: | [ { "host": "%%host%%", "port": 9095, "conf": [ { "include": { "domain": "kafka.connect", "type": "connector-task-metrics", "bean_regex": [ "kafka.connect:type=connector-task-metrics,connector=.*,task=.*" ], "attribute": { "batch-size-max": { "alias": "jmx.kafka.connect.connector.batch_size_max" }, "status": { "metric_type": "gauge", "alias": "jmx.kafka.connect.connector.status", "values": { "running":0, "paused":1, "failed":2, "destroyed":3, "unassigned":-1 } }, "batch-size-avg": { "alias": "jmx.kafka.connect.connector.batch_size_avg" }, "offset-commit-avg-time-ms": { "alias": "jmx.kafka.connect.connector.offset_commit_avg_time" }, "offset-commit-max-time-ms": { "alias": "jmx.kafka.connect.connector.offset_commit_max_time" }, "offset-commit-failure-percentage": { "alias": "jmx.kafka.connect.connector.offset_commit_failure_percentage" } } } }, { "include": { "domain": "kafka.connect", "type": "source-task-metrics", "bean_regex": [ "kafka.connect:type=source-task-metrics,connector=.*,task=.*" ], "attribute": { "source-record-poll-rate": { "alias": "jmx.kafka.connect.task.source_record_poll_rate" }, "source-record-write-rate": { "alias": "jmx.kafka.connect.task.source_record_write_rate" }, "poll-batch-avg-time-ms": { "alias": "jmx.kafka.connect.task.poll_batch_avg_time" }, "source-record-active-count-avg": { "alias": "jmx.kafka.connect.task.source_record_active_count_avg" }, "source-record-write-total": { "alias": "jmx.kafka.connect.task.source_record_write_total" }, "source-record-poll-total": { "alias": "jmx.kafka.connect.task.source_record_poll_total" } } } } ] } ]