| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Switch Down
expr: up{job=~"upTime|dmc1002"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} not responding for more than 1 minutes'
|
ok
|
|
27.154s ago
|
6.717ms |
| alert: Squid Server Down
expr: squid_server_up == 0
for: 10m
labels:
severity: critical
annotations:
summary: Squid on {{ $labels.instance }} not responding for more than 10 minutes
|
ok
|
|
27.148s ago
|
293us |
| alert: Bind Server Down
expr: bind_up == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.job }} on {{ $labels.instance }} not responding for more than 5 minutes'
|
ok
|
|
27.147s ago
|
299us |
| alert: Instance Down
expr: up{job=~"squid|wpad|etcd|dhcpd6|bind"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.job }} {{ $labels.instance }} not responding for more than 5 minutes.'
|
ok
|
|
27.147s ago
|
730us |
| alert: Error in dhcpserv config
expr: rate(dhcpserv_build_errors[1m]) > 1
for: 2m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} have too many error in dhcpserv generate config: {{ $value }}'
|
ok
|
|
27.147s ago
|
317.9us |
| alert: Cisco Netflow Cache High
expr: netflow_active_entries:ratio > 0.8
for: 5m
labels:
severity: ctritical
|
ok
|
|
27.147s ago
|
185.8us |
| alert: Postfix Queue High
expr: postfix_showq_message_size_bytes_count > 200
for: 10m
labels:
severity: critical
annotations:
summary: Postfix query overflow. Query exceeded {{ $value }} mails
|
ok
|
|
27.147s ago
|
314.9us |
| alert: Borg Backup Missing
expr: time() - borgbackup_last_modified > 86400 * 8
for: 30m
labels:
severity: critical
annotations:
summary: Backup on {{ $labels.instance }} not created last 3 days
|
ok
|
|
27.147s ago
|
572.1us |
| alert: Omada device down
expr: omada_device_uptime_seconds == 0
for: 5m
labels:
severity: critical
annotations:
summary: Device {{ $labels.device_type }} {{ $labels.device }} from site {{ $labels.site }} is down
|
ok
|
|
27.146s ago
|
3.384ms |
| alert: Unifi device down
expr: unifipoller_device_uptime_seconds == 0
for: 5m
labels:
severity: critical
annotations:
summary: Device {{ $labels.type }} {{ $labels.name }} from site {{ $labels.site_name }} is down
|
ok
|
|
27.143s ago
|
3.021ms |
| alert: Nginx is down
expr: nginx_up == 0
for: 5m
labels:
severity: critical
annotations:
summary: Nginx is down on the host {{ $labels.instance }} for more than 5 minutes
|
ok
|
|
27.14s ago
|
400.2us |
| alert: Nginx not all connections are handled
expr: rate(nginx_connections_handled{job="nginx"}[5m]) / rate(nginx_connections_accepted{job="nginx"}[5m]) < 1
for: 3m
labels:
severity: critical
annotations:
summary: Nginx does not handle all accept connections on the host {{ $labels.instance }} for more than 3 minutes
|
ok
|
|
27.14s ago
|
743.7us |
| alert: Nginx high active connections
expr: nginx_connections_active{job="nginx"} > 300
for: 5m
labels:
severity: critical
annotations:
summary: Большое коллчиество активных подключений в Nginx на {{ $labels.instance }} в течении 5 минут
|
ok
|
|
27.14s ago
|
375.6us |
| alert: Certificate expired
expr: nginx_cert_exporter_file_expired - time() < 86400 * 25
for: 12h
labels:
severity: critical
annotations:
summary: Локальный сертификат {{ $labels.name }} на {{ $labels.instance }} просрочиться в течении 25 дней
|
ok
|
|
27.139s ago
|
2.314ms |
| alert: DDOS
expr: (rate(nginx_http_requests_total[5m]) / rate(nginx_http_requests_total[5m] offset 5m) > 5) and rate(nginx_http_requests_total[5m]) > 100
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} находится под DDOS атакой'
|
ok
|
|
27.137s ago
|
956.7us |
| alert: Fail2Ban DDOS
expr: avg by(instance, jail) (f2b_jail_banned_current) > 20
for: 5m
labels:
severity: critical
annotations:
summary: 'Слишком часто срабатывает правило {{ $labels.jail }}: {{ $value }} на сервере {{ $labels.instance }}'
|
ok
|
|
27.137s ago
|
312.6us |
|
9.115s ago |
3.315ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Node Down
expr: up{job=~"node"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Node {{ $labels.instance }} not responding for more than 1 minutes.
|
ok
|
|
21.433s ago
|
1.408ms |
| alert: Service Down
expr: node_systemd_unit_state{name!="dnf-makecache.service",state=~"failed"} == 1
for: 5m
labels:
severity: critical
annotations:
summary: Service {{ $labels.name }} is not responding for 5m on {{ $labels.instance }} with status {{ $labels.state }}
|
ok
|
|
21.432s ago
|
83.74ms |
| alert: Node Low Disk Space
expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes{mountpoint!~".mnt.*"} < 10 and on(instance, device, mountpoint) node_filesystem_readonly == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} low disk space. {{ $value }} percent of the {{ $labels.mountpoint }} is free left'
|
ok
|
|
21.348s ago
|
14.21ms |
| alert: Host Oom Kill Detected
expr: increase(node_vmstat_oom_kill[1m]) > 0
labels:
severity: warning
annotations:
summary: OOM kill detected on {{ $labels.instance }}
|
ok
|
|
21.335s ago
|
821.7us |
| alert: Node Out Of Memory
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 20
for: 5m
labels:
severity: critical
annotations:
summary: 'Node {{ $labels.instance }} memory is filling up (< 20% left). Free now: {{ $value }}%'
|
ok
|
|
21.334s ago
|
1.468ms |
| alert: Node Memory Fill Up Soon
expr: predict_linear(node_memory_MemAvailable_bytes[2h], 1 * 3600) <= 0
for: 2h
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} memory will fiil up soon in 1h'
|
ok
|
|
21.333s ago
|
2.039ms |
| alert: Node SWAP Out Of Memory
expr: node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes * 100 < 20
for: 10m
labels:
severity: critical
annotations:
summary: 'Node {{ $labels.instance }} swap memory is filing up (< 20% left). Free now: {{ $value }}%'
|
ok
|
|
21.331s ago
|
1.982ms |
| alert: Node Memory Under Pressure
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100) < 20 and rate(node_vmstat_pgmajfault[5m]) > 100
for: 5m
labels:
severity: critical
annotations:
summary: Node {{ $labels.instance }} is under heavy memory pressure. High rate of major page faults. VALUE = {{ $value }}
|
ok
|
|
21.33s ago
|
1.535ms |
| alert: Node Unusual Network ThroughputIn
expr: sum by(instance) (rate(node_network_receive_bytes_total[30m])) / 1024 / 1024 > 100
for: 15m
labels:
severity: critical
annotations:
summary: Host unusual network throughput in {{ $labels.instance }} - {{ $value }}
|
ok
|
|
21.329s ago
|
2.476ms |
| alert: Node Unusual Network ThroughputOut
expr: sum by(instance) (rate(node_network_transmit_bytes_total[30m])) / 1024 / 1024 > 100
for: 15m
labels:
severity: critical
annotations:
summary: Host unusual network throughput out {{ $labels.instance }} - {{ $value }}
|
ok
|
|
21.327s ago
|
2.4ms |
| alert: Node Unusual Disk Read Rate
expr: sum by(instance) (rate(node_disk_read_bytes_total[30m])) / 1024 / 1024 > 200
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk read rate last 30 minutes {{ $labels.instance }} - {{ $value }} mb/s
|
ok
|
|
21.325s ago
|
3.093ms |
| alert: Node Unusual Disk Write Rate
expr: sum by(instance) (rate(node_disk_written_bytes_total[30m])) / 1024 / 1024 > 200
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk write rate last 30 minutes {{ $labels.instance }} - {{ $value }} mb/s
|
ok
|
|
21.322s ago
|
3.092ms |
| alert: Node Unusual Disk ReadLatency
expr: rate(node_disk_read_time_seconds_total[30m]) / rate(node_disk_reads_completed_total[30m]) > 0.3 and rate(node_disk_reads_completed_total[30m]) > 0
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk read latency {{ $labels.instance }} - {{ $value }} s
|
ok
|
|
21.319s ago
|
8.955ms |
| alert: Node Unusual Disk WriteLatency
expr: rate(node_disk_write_time_seconds_total[30m]) / rate(node_disk_writes_completed_total[30m]) > 0.15 and rate(node_disk_writes_completed_total[30m]) > 0
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk write latency {{ $labels.instance }} - {{ $value }} s
|
ok
|
|
21.311s ago
|
9.382ms |
| alert: Big IOWait
expr: avg by(instance) (irate(node_cpu_seconds_total{mode="iowait"}[1m])) * 100 > 15
for: 2m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} have big iowait query: {{ $value }}%'
|
ok
|
|
21.302s ago
|
7.753ms |
| alert: Node High Cpu Load
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{job="node",mode="idle"}[15m])) * 100) > 80
for: 15m
labels:
severity: critical
annotations:
summary: Host high CPU load {{ $labels.instance }} - {{ $value }}%
|
ok
|
|
21.295s ago
|
7.485ms |
| alert: Node Physical Component TooHot
expr: node_hwmon_temp_celsius{chip!~"nvme.+"} > 75
for: 10m
labels:
severity: critical
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }}) {{ $labels.chip }} - {{ $value }}C
|
ok
|
|
21.287s ago
|
3.403ms |
| alert: Node NVMe Drive TooHot
expr: node_hwmon_temp_celsius{chip=~"nvme.+"} > 100
for: 10m
labels:
severity: critical
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }}) {{ $labels.chip }} - {{ $value }}C
|
ok
|
|
21.284s ago
|
310.9us |
| alert: Node Overtemperature Alarm
expr: node_hwmon_temp_crit_alarm_celsius == 1
labels:
severity: critical
annotations:
summary: Host node overtemperature alarm (instance {{ $labels.instance }}) - {{ $value }}
|
ok
|
|
21.284s ago
|
2.537ms |
| alert: Node Raid Array Got Inactive
expr: node_md_state{state="inactive"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically
|
ok
|
|
21.282s ago
|
479.8us |
| alert: Node Raid Disk Failure
expr: node_md_disks{state="failed"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap
|
ok
|
|
21.282s ago
|
486.8us |
| alert: Node Edac Correctable Errors Detected
expr: increase(node_edac_correctable_errors_total[1m]) > 0
for: 10m
labels:
severity: critical
annotations:
summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }}) {{ $value }}
|
ok
|
|
21.282s ago
|
995.4us |
| alert: Node Edac Uncorrectable Errors Detected
expr: node_edac_uncorrectable_errors_total > 0
labels:
severity: critical
annotations:
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }}) {{ $value }}
|
ok
|
|
21.281s ago
|
775.7us |
| alert: Relocated Sectors
expr: smart_attribute_raw{attribute_id=~"197|198"} > 0
for: 3h
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} имеет {{ $value }} переназначенных секторов. {{ $labels.device_model_family }} {{ $labels.device_model_name }} под именем {{ $labels.device_name }} с серийным номером {{ $labels.device_serial_number }}'
|
ok
|
|
21.281s ago
|
2.542ms |
| alert: Node Clock Not Synchronising
expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
for: 2m
labels:
severity: critical
annotations:
summary: Clock not synchronising. Ensure NTP is configured on this host {{ $labels.instance }}
|
ok
|
|
21.278s ago
|
1.277ms |
| alert: HostClockSkew
expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
for: 2m
labels:
severity: warning
annotations:
summary: Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host {{ $labels.instance }}
|
ok
|
|
21.277s ago
|
2.077ms |
| alert: Node Disk Is Missing
expr: node_btrfs_device_size_bytes == 0
for: 1m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has a disk {{ $labels.device }} that is not in the file system'
|
ok
|
|
21.275s ago
|
2.002ms |
|
25.61s ago |
8.214ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Prometheus Configuration Failure
expr: prometheus_config_last_reload_successful != 1
labels:
severity: critical
annotations:
summary: Prometheus configuration reload failure (instance {{ $labels.instance }})
|
ok
|
|
25.61s ago
|
391.3us |
| alert: AlertManager Configuration Failure
expr: alertmanager_config_last_reload_successful != 1
labels:
severity: critical
annotations:
summary: AlertManager configuration reload failure (instance {{ $labels.iinstance }})
|
ok
|
|
25.61s ago
|
350us |
| alert: Prometheus NotConnected To Alertmanager
expr: prometheus_notifications_alertmanagers_discovered < 1
labels:
severity: critical
annotations:
summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }})
|
ok
|
|
25.61s ago
|
396.2us |
| alert: Prometheus Rule Evaluation Failures
expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
labels:
severity: critical
annotations:
summary: Prometheus rule evaluation failures (instance {{ $labels.instance }})
|
ok
|
|
25.609s ago
|
534.8us |
| alert: Prometheus Template Text Expansion Failures
expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0
labels:
severity: critical
annotations:
summary: Prometheus template text expansion failures (instance {{ $labels.instance }})
|
ok
|
|
25.609s ago
|
260.4us |
| alert: Prometheus Rule Evaluation Slow
expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds
for: 5m
labels:
severity: critical
annotations:
summary: Prometheus rule evaluation slow (instance {{ $labels.instance }})
|
ok
|
|
25.609s ago
|
779.9us |
| alert: Prometheus Target Empty
expr: prometheus_sd_discovered_targets == 0
labels:
severity: critical
annotations:
description: Prometheus has no target in service discovery
summary: Prometheus target empty (instance {{ $labels.instance }})
|
ok
|
|
25.608s ago
|
4.341ms |
| alert: Prometheus Large Scrape
expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10
for: 5m
labels:
severity: critical
annotations:
description: Prometheus has many scrapes that exceed the sample limit
summary: Prometheus large scrape (instance {{ $labels.instance }})
|
ok
|
|
25.604s ago
|
783.7us |
| alert: Prometheus Target Scrape Duplicate
expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0
labels:
severity: critical
annotations:
description: Prometheus has many samples rejected due to duplicate timestamps but different values
summary: Prometheus target scrape duplicate (instance {{ $labels.instance }})
|
ok
|
|
25.604s ago
|
335.6us |
|
15.536s ago |
7.198ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: UPS not response
expr: up{job=~"apcsnmp|upssnmp"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: UPS {{ $labels.instance }} not responding for more than 5 minutes.
|
ok
|
|
15.536s ago
|
1.849ms |
| alert: Server Room In FIRE!
expr: iemStatusProbeCurrentTemp{instance="Symmetra.ups"} > 24
for: 10m
labels:
severity: critical
annotations:
summary: 'Server room too hot: {{ $value }}C'
|
ok
|
|
15.534s ago
|
213.9us |
| alert: There is no ELECTRICITY in the Server Room!
expr: upsBasicInputPhase{instance="Symmetra.ups"} < 3
for: 5m
labels:
severity: critical
annotations:
summary: 'Working phases in Symmetra at the moment: {{ $value }}'
|
ok
|
|
15.534s ago
|
227.8us |
| alert: APC UPS too hot!
expr: iemStatusProbeCurrentTemp > 30
for: 5m
labels:
severity: critical
annotations:
summary: Too hot in {{ $labels.instance }}. Temperatur is {{ $value }}C
|
ok
|
|
15.534s ago
|
272.1us |
| alert: UPS too hot!
expr: upsBatteryTemperature > 45
for: 5m
labels:
severity: critical
annotations:
summary: Too hot in {{ $labels.instance }}. Temperatur is {{ $value }}C
|
ok
|
|
15.534s ago
|
1.378ms |
| alert: APC UPS low Input voltage
expr: upsAdvInputLineVoltage < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'APC UPS {{ $labels.instance }} has low voltage: {{ $value }}V'
|
ok
|
|
15.533s ago
|
303.2us |
| alert: UPS low Input voltage
expr: upsInputVoltage < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has low input voltage: {{ $value }}V'
|
ok
|
|
15.533s ago
|
608.5us |
| alert: UPS low Output voltage
expr: upsOutputVoltage{upsOutputLineIndex="1"} < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has low output voltage: {{ $value }}V'
|
ok
|
|
15.533s ago
|
640.3us |
| alert: UPS big load
expr: upsOutputPercentLoad{upsOutputLineIndex="1"} > 75
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has big load: {{ $value }}%'
|
ok
|
|
15.532s ago
|
575us |
| alert: Low APC UPS battery capacity
expr: upsAdvBatteryCapacity < 75
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has low battery capacity: {{ $value }}%'
|
ok
|
|
15.532s ago
|
199us |
| alert: Low UPS battery capacity
expr: upsEstimatedChargeRemaining < 75
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has low battery capacity: {{ $value }}%'
|
ok
|
|
15.532s ago
|
887.9us |
|
20.735s ago |
50.82ms |