prometheus整合jmx_exporter 使用jmx_exporter监控Kafka
docker-compose摆设kafka集群；单节点单zk-CSDN博客springboot整合kafka；docker摆设kafka-CSDN博客
kafka使用SSL加密和认证--todo_ssl.truststore.location-CSDN博客
version: '3.8'
services:
zookeeper1:
image: zookeeper:3.9.1
container_name: zookeeper1-cluster
hostname: zookeeper-cluster
ports:
- "2185:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_INIT_LIMIT: 10
ZOOKEEPER_SYNC_LIMIT: 5
volumes:
- ./zookeeper-data:/data
- ./zookeeper-datalog:/datalog
networks:
- kafka-cluster-network
kafka1:
image: bitnami/kafka:3.8
container_name: kafka1-cluster
depends_on:
- zookeeper1
ports:
- "9099:9092" # Kafka 客户端连接端口
- "9101:9101" # Prometheus 抓取指标的端口
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9099
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
KAFKA_CFG_FETCH_MAX_BYTES: 5242880
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9101:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
volumes:
- ./kafka1-data:/bitnami/kafka
- ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
- ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
networks:
- kafka-cluster-network
kafka2:
image: bitnami/kafka:3.8
container_name: kafka2-cluster
depends_on:
- zookeeper1
ports:
- "9096:9092" # Kafka 客户端连接端口
- "9102:9102" # Prometheus 抓取指标的端口
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9096
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
KAFKA_CFG_FETCH_MAX_BYTES: 5242880
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
volumes:
- ./kafka2-data:/bitnami/kafka
- ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
- ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
networks:
- kafka-cluster-network
kafka3:
image: bitnami/kafka:3.8
container_name: kafka3-cluster
depends_on:
- zookeeper1
ports:
- "9097:9092" # Kafka 客户端连接端口
- "9103:9103" # Prometheus 抓取指标的端口
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9097
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
KAFKA_CFG_FETCH_MAX_BYTES: 5242880
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9103:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
volumes:
- ./kafka3-data:/bitnami/kafka
- ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
- ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
networks:
- kafka-cluster-network
kafka-ui:
image: provectuslabs/kafka-ui:v0.7.2
container_name: kafka-ui-cluster
hostname: kafka-ui-cluster
ports:
- "9098:8080"
depends_on:
- kafka1
- kafka2
- kafka3
environment:
KAFKA_CLUSTERS_0_NAME: kafka-cluster
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 192.168.118.20:9096,192.168.118.20:9097,192.168.118.20:9099
KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-cluster:2181
networks:
- kafka-cluster-network
networks:
kafka-cluster-network:
driver: bridge
下载
jmx_prometheus_javaagent-0.20.0.jar
Central Repository: io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/
https://i-blog.csdnimg.cn/direct/8ef8787191a14346a0ffe147346b82ac.png
或者https://github.com/prometheus/jmx_exporter/releases/tag/0.20.0
https://i-blog.csdnimg.cn/direct/db3306e816b7403ea3d17237845a042a.png
下载 https://github.com/prometheus/jmx_exporter/tree/main/exampleskafka-jmx-exporter-config.yaml
https://i-blog.csdnimg.cn/direct/f946218475544dffb509b982c91229c1.pnghttps://github.com/prometheus/jmx_exporter/tree/main/examples
1、在启动kafka时配置代理
2、同时文件也挂载进去
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml" volumes:
- ./kafka2-data:/bitnami/kafka
- ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
- ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml - "9103:9103" # Prometheus 抓取指标的端口启动服务后 查抄一下 是否配置成功
http://192.168.118.20:9101/metrics
https://i-blog.csdnimg.cn/direct/1241e6d6939946e0b0ed585c2744b3cd.png
摆设prometheus 和 grafana
version: '3.8'
services:
prometheus:
image: bitnami/prometheus:3.0.0
container_name: prometheus
hostname: prometheus
ports:
- "9090:9090" # Prometheus Web UI 端口
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
- ./prometheus-data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-lifecycle'
- '--storage.tsdb.retention.time=90d'
- "--storage.tsdb.path=/prometheus"
- "--web.enable-admin-api"
restart: always
grafana:
image: grafana/grafana:11.3.3
container_name: grafana
hostname: grafana
ports:
- "3000:3000" # Grafana Web UI 端口
environment:
GF_SECURITY_ADMIN_PASSWORD: admin # 设置 Grafana 的管理员密码
volumes:
- ./grafana-storage:/var/lib/grafana
restart: always
prometheus.yml如下配置
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
- job_name: 'kafka'
static_configs:
- targets:
- '192.168.118.20:9101' # 对应 kafka1 的 JMX Exporter 端口
- '192.168.118.20:9102' # 对应 kafka2 的 JMX Exporter 端口
- '192.168.118.20:9103' # 对应 kafka3 的 JMX Exporter 端口
- job_name: "node"
static_configs:
- targets: ["localhost:9100"]grafana: 用户名和密码 都是 admin admin
http://192.168.xxx.xxx:3000/
https://i-blog.csdnimg.cn/direct/41555942f049495a99efe71065b7a4b4.png
https://i-blog.csdnimg.cn/direct/5611ee180f4a405d9c3a32d9fdc264db.png
https://grafana.com/grafana/dashboards/11962-kafka-metrics/
https://grafana.com/grafana/dashboards/11962-kafka-metrics/
https://i-blog.csdnimg.cn/direct/e1cdc773a94d4116977289039586ce8f.png
https://grafana.com/grafana/dashboards/10973-kafka-dashboard333/
https://i-blog.csdnimg.cn/direct/133946586478409787ce4981858efa01.png
Kafka Dashboard | Grafana Labs
该模版需要更新yaml
免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。
页:
[1]