本文是我根据日常Prometheus监控运维经验,列出的企业生成中常见的Prometheus服务监控告警规则。
目录
1、服务器告警规则 node_exporter_rules.yml
2、Radis服务告警规则 redis_exporter_rules.yml
3、RabbitMQ服务告警规则 rabbitmq_exporter_rules.yml
4、kafka集群服务告警规则 kafka_exporter_rules.yml
5、域名SSL证书过期监控规则 kafka_exporter_rules.yml文章来源:https://www.toymoban.com/news/detail-623931.html
6、Elasticsearch集群告警规则 Elasticsearch_exporter_rules.yml文章来源地址https://www.toymoban.com/news/detail-623931.html
1、服务器告警规则 node_exporter_rules.yml
[root@grafana rules]# cat node_exporter_rules.yml
# 服务器资源告警策略
groups:
- name: 服务器资源监控
rules:
- alert: 内存使用率过高
expr: 100 - (node_memory_MemFree_bytes+node_memory_Cached_bytes+node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 > 90
for: 5m # 告警持续时间,超过这个时间才会发送给alertmanager
labels:
severity: 严重告警
annotations:
summary: "{{ $labels.instance }} 内存使用率过高,请尽快处理!"
description: "{{ $labels.instance }}内存使用率超过90%,当前使用率{{ $value }}%."
- alert: 服务器宕机
expr: up == 0
for: 3m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 服务器宕机,请尽快处理!"
description: "{{$labels.instance}} 服务器延时超过3分钟,当前状态{{ $value }}. "
- alert: CPU高负荷
expr: 100 - (avg by (instance,job)(irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} CPU使用率过高,请尽快处理!"
description: "{{$labels.instance}} CPU使用大于90%,当前使用率{{ $value }}%. "
- alert: 磁盘IO性能
expr: avg(irate(node_disk_io_time_seconds_total[1m])) by(instance,job)* 100 > 90
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流入磁盘IO使用率过高,请尽快处理!"
description: "{{$labels.instance}} 流入磁盘IO大于90%,当前使用率{{ $value }}%."
- alert: 网络流入
expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流入网络带宽过高,请尽快处理!"
description: "{{$labels.instance}} 流入网络带宽持续5分钟高于100M. RX带宽使用量{{$value}}."
- alert: 网络流出
expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流出网络带宽过高,请尽快处理!"
description: "{{$labels.instance}} 流出网络带宽持续5分钟高于100M. RX带宽使用量{$value}}."
- alert: TCP连接数
expr: node_netstat_Tcp_CurrEstab > 10000
for: 2m
labels:
severity: 严重告警
annotations:
summary: " TCP_ESTABLISHED过高!"
description: "{{$labels.instance}} TCP_ESTABLISHED大于100%,当前使用率{{ $value }}%."
- alert: 磁盘容量
expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 90
for: 1m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 磁盘分区使用率过高,请尽快处理!"
description: "{{$labels.instance}} 磁盘分区使用大于90%,当前使用率{{ $value }}%."
2、Radis服务告警规则 redis_exporter_rules.yml
[root@grafana rules]# cat redis_exporter_rules.yml
# Redis服务监控
groups:
- name: Redis服务监控
rules:
- alert: Redis服务停止
expr: redis_up == 0
for: 1m
labels:
severity: 严重告警
annotations:
summary: " {{ $labels.alias }} Redis服务已停止,当前状态{{ $value }}"
description: "{{$labels.instance}}:Redis 服务停止运行 "
- alert: Redis连接数超过最大
expr: redis_connected_clients / redis_config_maxclients * 100 > 80
for: 1m
labels:
severity: 严重告警
annotations:
summary: " {{ $labels.alias }}"
description: " {{$labels.instance}}:Redis连接数超过最大连接数的80%.当前连接数{{ $value }}"
3、RabbitMQ服务告警规则 rabbitmq_exporter_rules.yml
[root@grafana rules]# cat rabbitmq_exporter_rules.yml
# RabbitMQ服务监控
groups:
- name: RabbitMQ服务监控
rules:
- alert: RabbitMQ服务停止
expr: rabbitmq_up ==0
for: 3m
labels:
severity: 严重告警
annotations:
description: "{{$labels.instance}}RabbitMQ服务已停止,当前状态{{ $value }}"
summary: "RabbitMQ服务已停止3分钟,请尽快处理!"
- alert: RabbitMQ内存使用大于2G
expr: rabbitmq_node_mem_used/1024/1024 > 2048
for: 3m
labels:
severity: 严重告警
annotations:
description: "{{ $labels.instance }} RabbitMQ内存使占用过高 !"
value: '{{ $value }} MB'
summary: "RabbitMQ内存使占用大于2G"
4、kafka集群服务告警规则 kafka_exporter_rules.yml
[root@grafana rules]# cat kafka_exporter_rules.yml
# kafka集群服务监控
groups:
- name: kafka服务监控
rules:
- alert: kafka消费滞后
expr: sum(kafka_consumergroup_lag{topic!="sop_free_study_fix-student_wechat_detail"}) by (consumergroup, topic, job) > 50000
for: 3m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} kafka消费滞后({{$.Labels.consumergroup}})"
description: "{{$.Labels.topic}}消费滞后超过5万持续3分钟(当前{{$value}})"
- alert: kafka集群节点减少
expr: kafka_brokers < 3 #kafka集群节点数3
for: 3m
labels:
severity: 严重告警
annotations:
summary: "kafka集群部分节点已停止,请尽快处理!"
description: "{{$labels.instance}} kafka集群节点减少"
- alert: emqx_rule_to_kafka最近五分钟内的每秒平均变化率为0
expr: sum(rate(kafka_topic_partition_current_offset{topic="emqx_rule_to_kafka"}[5m])) by ( instance,topic,job) ==0
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} emqx_rule_to_kafka未接收到消息"
description: "{{$.Labels.topic}}emqx_rule_to_kafka持续5分钟未接收到消息(当前{{$value}})"
5、域名SSL证书过期监控规则 kafka_exporter_rules.yml
[root@grafana rules]# cat ssl_expiry.yml
groups:
- name: SSL证书监测
rules:
- alert: 证书还有30天过期
expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 300
for: 5m
labels:
severity: 重要告警
annotations:
summary: "SSL证书即将过期 (instance {{ $labels.instance }})"
description: "SSL证书即将30天内过期 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: 证书已过期
expr: probe_ssl_earliest_cert_expiry - time() <= 0
for: 5m
labels:
severity: 严重告警
annotations:
summary: "SSL证书已经过期 (instance {{ $labels.instance }})"
description: "SSL证书已经过期\n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
6、Elasticsearch集群告警规则 Elasticsearch_exporter_rules.yml
[root@grafana rules]# cat elasticsearch_exporter_rules.yml
groups:
- name: ElasticSearch服务监控
rules:
- alert: ES集群节点减少
expr: elasticsearch_cluster_health_number_of_nodes < 3 #ES集群节点数3
for: 5m
labels:
severity: 严重告警
annotations:
summary: "ES集群节点减少:{{$.Labels.job}}"
description: "ES集群节点数减少:{{$.Labels.job}},(当前:{{$value}})"
- alert: jvm内存使用率告警
expr: elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}*100 > 90
for: 5m
labels:
severity: 严重告警
annotations:
summary: "jvm内存使用率过高:{{$.Labels.job}}"
description: "jvm内存使用率过高:{{$.Labels.job}}大于90%,(当前:{{$value}})"
到了这里,关于三、Prometheus常用告警规则rules.yml的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!