groups: - name: node_live rules: - alert: NodeInstanceDown expr: up{job=~".*node.*"} == 0 for: 30s labels: severity: critical annotations: summary: "服务器 {{ $labels.instance }} 监控失联" description: "节点 {{ $labels.instance }} node_exporter 离线,服务器宕机或进程挂掉" - name: cpu_alert rules: # CPU 使用率超过 85% 告警 - alert: CpuUsageHigh expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100) > 85 for: 2m labels: severity: warning annotations: summary: "服务器 {{ $labels.instance }} CPU 负载过高" description: "CPU 当前使用率:{{ $value }}%" - alert: CpuUsageCritical expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100) > 95 for: 1m labels: severity: critical annotations: summary: "服务器 {{ $labels.instance }} CPU 严重过载" description: "CPU 当前使用率:{{ $value }}%" - name: memory_alert rules: # 内存使用率 >85% 警告 - alert: MemoryUsageHigh expr: (1 - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes) * 100 > 85 for: 2m labels: severity: warning annotations: summary: "服务器 {{ $labels.instance }} 内存使用率过高" description: "内存当前使用率:{{ $value }}%" - alert: MemoryUsageCritical expr: (1 - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes) * 100 > 95 for: 1m labels: severity: critical annotations: summary: "服务器 {{ $labels.instance }} 内存严重不足" description: "内存当前使用率:{{ $value }}%" - name: disk_alert rules: # 专属 /serverhome 剩余小于100G 严重告警 - alert: ServerhomeDiskLess100G expr: node_filesystem_avail_bytes{mountpoint="/serverhome"} < 100 * 1024 * 1024 * 1024 for: 1m labels: severity: critical annotations: summary: "/serverhome 分区剩余空间不足100GB" description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 剩余:{{ $value }} 字节" # - alert: DiskAvailLess20G # expr: node_filesystem_avail_bytes{mountpoint!~"^/(sys|proc|dev|tmp|boot|boot/efi|run)($|/)"} < 20 * 1024 * 1024 * 1024 # for: 1m # labels: # severity: warning # annotations: # summary: "挂载点 {{ $labels.mountpoint }} 剩余空间不足20GB" # description: "节点 {{ $labels.instance }} 剩余:{{ $value }} 字节" # # - alert: DiskUsageOver90 # expr: (100 - (node_filesystem_avail_bytes{mountpoint!~"^/(sys|proc|dev|tmp|boot|boot/efi|run)($|/)"} / node_filesystem_size_bytes * 100)) > 90 and instance!="10.150.111.240:9100" and instance!="10.153.10.200:9100" # for: 2m # labels: # severity: critical # annotations: # summary: "挂载点 {{ $labels.mountpoint }} 磁盘使用率超过90%" # description: "节点 {{ $labels.instance }} 使用率:{{ $value }}%" - name: network_alert rules: # 网卡入流量 峰值过大 - alert: NetworkInHigh expr: avg by(instance,device) (irate(node_network_receive_bytes_total{device!~"lo"}[1m])) / 1024 / 1024 > 80 for: 1m labels: severity: warning annotations: summary: "服务器 {{ $labels.instance }} 网卡 {{ $labels.device }} 入流量过高" description: "当前入流量:{{ $value }} MB/s" # 网卡出流量 峰值过大 - alert: NetworkOutHigh expr: avg by(instance,device) (irate(node_network_transmit_bytes_total{device!~"lo"}[1m])) / 1024 / 1024 > 80 for: 1m labels: severity: warning annotations: summary: "服务器 {{ $labels.instance }} 网卡 {{ $labels.device }} 出流量过高" description: "当前出流量:{{ $value }} MB/s"