init prometheus监控配置文件
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
groups:
|
||||
- name: single_port_service_alarm
|
||||
rules:
|
||||
########################### 严重告警 Critical ###########################
|
||||
# 22 sshd 远程登录端口
|
||||
- alert: Port_22_SSHD_Down
|
||||
expr: probe_success{port="22"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
service: sshd
|
||||
annotations:
|
||||
summary: "主机{{$labels.instance}} SSH登录端口{{$labels.port}}断开"
|
||||
description: "端口{{$labels.port}} sshd服务异常，无法远程登录服务器，紧急处理！"
|
||||
|
||||
# 111 rpcbind RPC注册服务
|
||||
- alert: Port_111_Rpcbind_Down
|
||||
expr: probe_success{port="111"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
service: rpcbind
|
||||
annotations:
|
||||
summary: "主机{{$labels.instance}} RPC注册端口{{$labels.port}}断开"
|
||||
description: "端口{{$labels.port}} rpcbind异常，所有NFS/RPC相关服务全部失效，业务挂载会报错！"
|
||||
|
||||
# 2049 NFS存储端口
|
||||
- alert: Port_2049_NFS_Down
|
||||
expr: probe_success{port="2049"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
service: nfs
|
||||
annotations:
|
||||
summary: "主机{{$labels.instance}} NFS存储端口{{$labels.port}}断开"
|
||||
description: "端口{{$labels.port}} NFS内核端口无法连通，所有业务NFS读写中断！"
|
||||
|
||||
# 9100 node_exporter监控采集
|
||||
- alert: Port_9100_NodeExporter_Down
|
||||
expr: probe_success{port="9100"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
service: node_exporter
|
||||
annotations:
|
||||
summary: "主机{{$labels.instance}} 监控采集端口{{$labels.port}}断开"
|
||||
description: "端口{{$labels.port}} node_exporter离线，主机资源指标无法采集，无监控数据！"
|
||||
|
||||
########################### 通用端口延迟告警 ###########################
|
||||
- alert: Port_High_Latency
|
||||
expr: probe_duration_seconds > 0.5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "主机{{$labels.instance}} {{$labels.service}}端口{{$labels.port}}连接延迟过高"
|
||||
description: "端口{{$labels.port}}服务{{$labels.service}}网络耗时超过500ms，网络存在拥塞"
|
||||
@@ -0,0 +1,133 @@
|
||||
groups:
|
||||
- name: freeipa_alerts
|
||||
rules:
|
||||
# 1. FreeIPA 节点宕机 / node_exporter 挂了
|
||||
- alert: FreeIPA_Node_Down
|
||||
expr: up{job=~"freeipa|ipa"} == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "FreeIPA 节点失联"
|
||||
description: "节点 {{ $labels.instance }} node-exporter 掉线超过30秒，主机不可达"
|
||||
|
||||
# 2. FreeIPA 核心综合服务异常（krb5kdc/httpd/ipa-custodia/pki-tomcatd）
|
||||
- alert: FreeIPA_Service_Down
|
||||
expr: sum by(instance,name) (node_systemd_unit_state{name=~"krb5kdc.service|httpd.service|ipa-custodia.service|pki-tomcatd@.*.service",state="active"}) != 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "FreeIPA 核心服务异常停止"
|
||||
description: "节点 {{ $labels.instance }} 服务 {{ $labels.name }} 未处于运行状态"
|
||||
|
||||
# 3. Kerberos KDC 认证服务异常
|
||||
- alert: FreeIPA_Kerberos_Down
|
||||
expr: sum by(instance,name) (node_systemd_unit_state{name="krb5kdc.service",state="active"}) != 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Kerberos KDC 认证服务异常"
|
||||
description: "krb5kdc.service 停止，域用户无法完成Kerberos认证登录"
|
||||
|
||||
# 4. FreeIPA Web 管理界面 httpd 服务异常
|
||||
- alert: FreeIPA_HTTP_Down
|
||||
expr: sum by(instance,name) (node_systemd_unit_state{name="httpd.service",state="active"}) != 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "FreeIPA Web 管理界面服务异常"
|
||||
description: "httpd.service 已停止，无法访问IPA网页管理端"
|
||||
|
||||
# 5. PKI 证书签发服务 pki-tomcatd 异常
|
||||
- alert: FreeIPA_PKI_Tomcat_Down
|
||||
expr: sum by(instance,name) (node_systemd_unit_state{name=~"pki-tomcatd@.*.service",state="active"}) != 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "FreeIPA PKI 证书服务异常"
|
||||
description: "pki-tomcatd 证书签发服务停止，新证书无法申请下发"
|
||||
|
||||
# 6. 专属监控 dirsrv@ADVANTEST-COM.service 389 LDAP目录服务停机
|
||||
- alert: DirsrvServiceDown
|
||||
expr: sum by(instance,name) (node_systemd_unit_state{name="dirsrv@ADVANTEST-COM.service",state="active"}) != 1
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: dirsrv-ADVANTEST
|
||||
annotations:
|
||||
summary: "389 LDAP目录服务已停止"
|
||||
description: "主机{{ $labels.instance }} dirsrv@ADVANTEST-COM.service 非运行状态，LDAP查询/认证失效，请立即检查"
|
||||
|
||||
# 7. LDAP 端口 389/636 探测不通
|
||||
- alert: FreeIPA_LDAP_Port_Down
|
||||
expr: probe_tcp_port{target=~".*:389|.*:636"} == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "LDAP 端口连通性异常"
|
||||
description: "无法TCP连接 {{ $labels.target }}，防火墙/服务异常"
|
||||
|
||||
# 8. Kerberos 88端口探测不通
|
||||
- alert: FreeIPA_KRB5_Port_Down
|
||||
expr: probe_tcp_port{target=~".*:88"} == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Kerberos 88端口不通"
|
||||
description: "目标 {{ $labels.target }} TCP88端口无法访问，Kerberos认证阻断"
|
||||
|
||||
# 9. 证书剩余 <30天 预警
|
||||
- alert: FreeIPA_Certificate_Will_Expire
|
||||
expr: ipa_cert_expiry_hours < 720
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "FreeIPA 证书即将过期（剩余<30天）"
|
||||
description: "证书 {{ $labels.cert }} 有效期剩余不足30天，请规划续期"
|
||||
|
||||
# 10. 证书剩余 <7天 紧急告警
|
||||
- alert: FreeIPA_Certificate_Will_Expire_Critical
|
||||
expr: ipa_cert_expiry_hours < 168
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "FreeIPA 证书即将过期！紧急处理"
|
||||
description: "证书 {{ $labels.cert }} 有效期剩余不足7天，逾期会导致域认证全部失效"
|
||||
|
||||
# 11. 内存使用率超过85%
|
||||
- alert: FreeIPA_High_Memory_Usage
|
||||
expr: 100 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "FreeIPA主机内存使用率过高"
|
||||
description: "{{ $labels.instance }} 内存使用率 > 85%，存在OOM风险"
|
||||
|
||||
# 12. CPU使用率超过90%
|
||||
- alert: FreeIPA_High_CPU_Usage
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "FreeIPA主机CPU使用率过高"
|
||||
description: "{{ $labels.instance }} CPU持续占用高于90%，业务响应缓慢"
|
||||
|
||||
# 13. 根分区磁盘使用率超85%
|
||||
- alert: FreeIPA_High_Disk_Usage
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes * 100) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "主机根分区磁盘使用率过高"
|
||||
description: "{{ $labels.instance }} / 分区磁盘占用 > 85%，请清理日志/扩容"
|
||||
@@ -0,0 +1,104 @@
|
||||
groups:
|
||||
- name: node_live
|
||||
rules:
|
||||
- alert: NodeInstanceDown
|
||||
expr: up{job=~".*node.*"} == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} 监控失联"
|
||||
description: "节点 {{ $labels.instance }} node_exporter 离线，服务器宕机或进程挂掉"
|
||||
|
||||
- name: cpu_alert
|
||||
rules:
|
||||
# CPU 使用率超过 85% 告警
|
||||
- alert: CpuUsageHigh
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} CPU 负载过高"
|
||||
description: "CPU 当前使用率：{{ $value }}%"
|
||||
|
||||
- alert: CpuUsageCritical
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100) > 95
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} CPU 严重过载"
|
||||
description: "CPU 当前使用率：{{ $value }}%"
|
||||
|
||||
- name: memory_alert
|
||||
rules:
|
||||
# 内存使用率 >85% 警告
|
||||
- alert: MemoryUsageHigh
|
||||
expr: (1 - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes) * 100 > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} 内存使用率过高"
|
||||
description: "内存当前使用率：{{ $value }}%"
|
||||
|
||||
- alert: MemoryUsageCritical
|
||||
expr: (1 - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes) * 100 > 95
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} 内存严重不足"
|
||||
description: "内存当前使用率：{{ $value }}%"
|
||||
|
||||
- name: disk_alert
|
||||
rules:
|
||||
# 专属 /serverhome 剩余小于100G 严重告警
|
||||
- alert: ServerhomeDiskLess100G
|
||||
expr: node_filesystem_avail_bytes{mountpoint="/serverhome"} < 100 * 1024 * 1024 * 1024
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "/serverhome 分区剩余空间不足100GB"
|
||||
description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 剩余：{{ $value }} 字节"
|
||||
|
||||
# - alert: DiskAvailLess20G
|
||||
# expr: node_filesystem_avail_bytes{mountpoint!~"^/(sys|proc|dev|tmp|boot|boot/efi|run)($|/)"} < 20 * 1024 * 1024 * 1024
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: "挂载点 {{ $labels.mountpoint }} 剩余空间不足20GB"
|
||||
# description: "节点 {{ $labels.instance }} 剩余：{{ $value }} 字节"
|
||||
#
|
||||
# - alert: DiskUsageOver90
|
||||
# expr: (100 - (node_filesystem_avail_bytes{mountpoint!~"^/(sys|proc|dev|tmp|boot|boot/efi|run)($|/)"} / node_filesystem_size_bytes * 100)) > 90 and instance!="10.150.111.240:9100" and instance!="10.153.10.200:9100"
|
||||
# for: 2m
|
||||
# labels:
|
||||
# severity: critical
|
||||
# annotations:
|
||||
# summary: "挂载点 {{ $labels.mountpoint }} 磁盘使用率超过90%"
|
||||
# description: "节点 {{ $labels.instance }} 使用率：{{ $value }}%"
|
||||
- name: network_alert
|
||||
rules:
|
||||
# 网卡入流量 峰值过大
|
||||
- alert: NetworkInHigh
|
||||
expr: avg by(instance,device) (irate(node_network_receive_bytes_total{device!~"lo"}[1m])) / 1024 / 1024 > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} 网卡 {{ $labels.device }} 入流量过高"
|
||||
description: "当前入流量：{{ $value }} MB/s"
|
||||
|
||||
# 网卡出流量 峰值过大
|
||||
- alert: NetworkOutHigh
|
||||
expr: avg by(instance,device) (irate(node_network_transmit_bytes_total{device!~"lo"}[1m])) / 1024 / 1024 > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "服务器 {{ $labels.instance }} 网卡 {{ $labels.device }} 出流量过高"
|
||||
description: "当前出流量：{{ $value }} MB/s"
|
||||
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: tcp_port_down_rules
|
||||
rules:
|
||||
- alert: NIS Service Down
|
||||
expr: probe_success{port="111"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
port: "111"
|
||||
annotations:
|
||||
summary: "主机 {{$labels.instance}} RPC 111端口不可达"
|
||||
description: "111(rpcbind/NIS)端口断开，NIS/NFS依赖异常，实例：{{$labels.instance}}，端口：111"
|
||||
|
||||
- alert: NFS Service Down
|
||||
expr: probe_success{port="2049"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
port: "2049"
|
||||
annotations:
|
||||
summary: "主机 {{$labels.instance}} NFS 2049端口不可达"
|
||||
description: "2049(NFS服务)端口断开，NFS挂载访问失败，实例：{{$labels.instance}}，端口：2049"
|
||||
- alert: 389 Port Down
|
||||
expr: probe_success{port="389"} == 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
port: "389"
|
||||
annotations:
|
||||
summary: "主机 {{$labels.instance}} 认证 389端口不可达"
|
||||
@@ -0,0 +1,88 @@
|
||||
groups:
|
||||
- name: Linux-Base-Alert
|
||||
rules:
|
||||
|
||||
# 节点离线 = Failed
|
||||
- alert: Host_Down_Failed
|
||||
expr: up == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
level: Failed
|
||||
annotations:
|
||||
summary: "❌ 节点离线 Failed"
|
||||
description: |
|
||||
主机：{{ $labels.instance }}
|
||||
任务：{{ $labels.job }}
|
||||
- alert: Host_Ping_Offline
|
||||
expr: probe_success{job=~"blackbox_icmp.*"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
level: Down
|
||||
annotations:
|
||||
summary: "主机 Ping 失联 {{ $labels.instance }}"
|
||||
description: "主机网络不通，ICMP探测失败，大概率整机宕机或网段隔离"
|
||||
|
||||
- alert: Port_Down
|
||||
expr: probe_success{port!=""} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
level: Down
|
||||
annotations:
|
||||
summary: "端口 {{ $labels.port }} 通信异常 {{ $labels.instance }}"
|
||||
description: |
|
||||
业务：{{ $labels.business }}
|
||||
监控任务：{{ $labels.job }}
|
||||
故障地址：{{ $labels.instance }}
|
||||
异常端口：{{ $labels.port }}
|
||||
|
||||
# CPU 警告 Warning
|
||||
- alert: CPU_High_Warning
|
||||
expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
level: Warning
|
||||
annotations:
|
||||
summary: "⚠️ CPU 负载过高 Warning"
|
||||
description: "主机 {{ $labels.instance }} CPU 使用率：{{ printf \"%.2f\" $value }} %"
|
||||
|
||||
# 内存严重 Failed
|
||||
- alert: Mem_High_Failed
|
||||
expr: 100 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100) > 90
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
level: Failed
|
||||
annotations:
|
||||
summary: "❌ 内存占用过高 Failed"
|
||||
description: "主机 {{ $labels.instance }} 内存使用率：{{ printf \"%.2f\" $value }} %"
|
||||
|
||||
# 磁盘警告 Warning
|
||||
- alert: Disk_High_Warning
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes * 100) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
level: Warning
|
||||
annotations:
|
||||
summary: "⚠️ 磁盘空间不足 Warning"
|
||||
description: "分区 {{ $labels.mountpoint }} 使用率：{{ printf \"%.2f\" $value }} %"
|
||||
|
||||
- name: FreeIPA-Alert
|
||||
rules:
|
||||
# FreeIPA 核心服务异常 Failed
|
||||
- alert: FreeIPA_Service_Failed
|
||||
expr: node_systemd_unit_state{name=~"ipa|dirsrv|krb5kdc|httpd|pki-tomcatd|sssd",state="active"} == 0
|
||||
for: 60s
|
||||
labels:
|
||||
severity: critical
|
||||
level: Failed
|
||||
annotations:
|
||||
summary: "❌ FreeIPA 服务异常 Failed"
|
||||
description: |
|
||||
故障主机：{{ $labels.instance }}
|
||||
异常服务：{{ $labels.name }}
|
||||
影响：域认证、LDAP、登录挂载异常
|
||||
Reference in New Issue
Block a user