| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Switch Down
expr: up{job=~"upTime|dmc1002"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} not responding for more than 1 minutes'
|
ok
|
|
2.368s ago
|
7.587ms |
| alert: Squid Server Down
expr: squid_server_up == 0
for: 10m
labels:
severity: critical
annotations:
summary: Squid on {{ $labels.instance }} not responding for more than 10 minutes
|
ok
|
|
2.36s ago
|
249.5us |
| alert: Bind Server Down
expr: bind_up == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.job }} on {{ $labels.instance }} not responding for more than 5 minutes'
|
ok
|
|
2.36s ago
|
265.8us |
| alert: Instance Down
expr: up{job=~"squid|wpad|etcd|dhcpd6|bind"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.job }} {{ $labels.instance }} not responding for more than 5 minutes.'
|
ok
|
|
2.36s ago
|
514.9us |
| alert: Error in dhcpserv config
expr: rate(dhcpserv_build_errors[1m]) > 1
for: 2m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} have too many error in dhcpserv generate config: {{ $value }}'
|
ok
|
|
2.36s ago
|
323.8us |
| alert: Cisco Netflow Cache High
expr: netflow_active_entries:ratio > 0.8
for: 5m
labels:
severity: ctritical
|
ok
|
|
2.36s ago
|
194.1us |
| alert: Postfix Queue High
expr: postfix_showq_message_size_bytes_count > 200
for: 10m
labels:
severity: critical
annotations:
summary: Postfix query overflow. Query exceeded {{ $value }} mails
|
ok
|
|
2.36s ago
|
315.9us |
| alert: Borg Backup Missing
expr: time() - borgbackup_last_modified > 86400 * 8
for: 30m
labels:
severity: critical
annotations:
summary: Backup on {{ $labels.instance }} not created last 3 days
|
ok
|
|
2.36s ago
|
575.8us |
| alert: Omada device down
expr: omada_device_uptime_seconds == 0
for: 5m
labels:
severity: critical
annotations:
summary: Device {{ $labels.device_type }} {{ $labels.device }} from site {{ $labels.site }} is down
|
ok
|
|
2.359s ago
|
2.735ms |
| alert: Unifi device down
expr: unifipoller_device_uptime_seconds == 0
for: 5m
labels:
severity: critical
annotations:
summary: Device {{ $labels.type }} {{ $labels.name }} from site {{ $labels.site_name }} is down
|
ok
|
|
2.357s ago
|
4.247ms |
| alert: Nginx is down
expr: nginx_up == 0
for: 5m
labels:
severity: critical
annotations:
summary: Nginx is down on the host {{ $labels.instance }} for more than 5 minutes
|
ok
|
|
2.353s ago
|
486.7us |
| alert: Nginx not all connections are handled
expr: rate(nginx_connections_handled{job="nginx"}[5m]) / rate(nginx_connections_accepted{job="nginx"}[5m]) < 1
for: 3m
labels:
severity: critical
annotations:
summary: Nginx does not handle all accept connections on the host {{ $labels.instance }} for more than 3 minutes
|
ok
|
|
2.353s ago
|
945.4us |
| alert: Nginx high active connections
expr: nginx_connections_active{job="nginx"} > 300
for: 5m
labels:
severity: critical
annotations:
summary: Большое коллчиество активных подключений в Nginx на {{ $labels.instance }} в течении 5 минут
|
ok
|
|
2.352s ago
|
325.8us |
| alert: Certificate expired
expr: nginx_cert_exporter_file_expired - time() < 86400 * 25
for: 12h
labels:
severity: critical
annotations:
summary: Локальный сертификат {{ $labels.name }} на {{ $labels.instance }} просрочиться в течении 25 дней
|
ok
|
|
2.352s ago
|
2.544ms |
| alert: DDOS
expr: (rate(nginx_http_requests_total[5m]) / rate(nginx_http_requests_total[5m] offset 5m) > 5) and rate(nginx_http_requests_total[5m]) > 100
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} находится под DDOS атакой'
|
ok
|
|
2.349s ago
|
980.4us |
| alert: Fail2Ban DDOS
expr: avg by(instance, jail) (f2b_jail_banned_current) > 20
for: 5m
labels:
severity: critical
annotations:
summary: 'Слишком часто срабатывает правило {{ $labels.jail }}: {{ $value }} на сервере {{ $labels.instance }}'
|
ok
|
|
2.349s ago
|
411us |
|
14.329s ago |
3.217ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Node Down
expr: up{job=~"node"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Node {{ $labels.instance }} not responding for more than 1 minutes.
|
ok
|
|
26.646s ago
|
1.29ms |
| alert: Service Down
expr: node_systemd_unit_state{name!="dnf-makecache.service",state=~"failed"} == 1
for: 5m
labels:
severity: critical
annotations:
summary: Service {{ $labels.name }} is not responding for 5m on {{ $labels.instance }} with status {{ $labels.state }}
|
ok
|
|
26.645s ago
|
85.16ms |
| alert: Node Low Disk Space
expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes{mountpoint!~".mnt.*"} < 10 and on(instance, device, mountpoint) node_filesystem_readonly == 0
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} low disk space. {{ $value }} percent of the {{ $labels.mountpoint }} is free left'
|
ok
|
|
26.56s ago
|
8.108ms |
| alert: Host Oom Kill Detected
expr: increase(node_vmstat_oom_kill[1m]) > 0
labels:
severity: warning
annotations:
summary: OOM kill detected on {{ $labels.instance }}
|
ok
|
|
26.552s ago
|
392us |
| alert: Node Out Of Memory
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 20
for: 5m
labels:
severity: critical
annotations:
summary: 'Node {{ $labels.instance }} memory is filling up (< 20% left). Free now: {{ $value }}%'
|
ok
|
|
26.552s ago
|
1.586ms |
| alert: Node Memory Fill Up Soon
expr: predict_linear(node_memory_MemAvailable_bytes[2h], 1 * 3600) <= 0
for: 2h
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} memory will fiil up soon in 1h'
|
ok
|
|
26.551s ago
|
2.747ms |
| alert: Node SWAP Out Of Memory
expr: node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes * 100 < 20
for: 10m
labels:
severity: critical
annotations:
summary: 'Node {{ $labels.instance }} swap memory is filing up (< 20% left). Free now: {{ $value }}%'
|
ok
|
|
26.548s ago
|
2.179ms |
| alert: Node Memory Under Pressure
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100) < 20 and rate(node_vmstat_pgmajfault[5m]) > 100
for: 5m
labels:
severity: critical
annotations:
summary: Node {{ $labels.instance }} is under heavy memory pressure. High rate of major page faults. VALUE = {{ $value }}
|
ok
|
|
26.546s ago
|
2.338ms |
| alert: Node Unusual Network ThroughputIn
expr: sum by(instance) (rate(node_network_receive_bytes_total[30m])) / 1024 / 1024 > 100
for: 15m
labels:
severity: critical
annotations:
summary: Host unusual network throughput in {{ $labels.instance }} - {{ $value }}
|
ok
|
|
26.544s ago
|
5.011ms |
| alert: Node Unusual Network ThroughputOut
expr: sum by(instance) (rate(node_network_transmit_bytes_total[30m])) / 1024 / 1024 > 100
for: 15m
labels:
severity: critical
annotations:
summary: Host unusual network throughput out {{ $labels.instance }} - {{ $value }}
|
ok
|
|
26.539s ago
|
4.708ms |
| alert: Node Unusual Disk Read Rate
expr: sum by(instance) (rate(node_disk_read_bytes_total[30m])) / 1024 / 1024 > 200
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk read rate last 30 minutes {{ $labels.instance }} - {{ $value }} mb/s
|
ok
|
|
26.535s ago
|
5.948ms |
| alert: Node Unusual Disk Write Rate
expr: sum by(instance) (rate(node_disk_written_bytes_total[30m])) / 1024 / 1024 > 200
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk write rate last 30 minutes {{ $labels.instance }} - {{ $value }} mb/s
|
ok
|
|
26.529s ago
|
6.324ms |
| alert: Node Unusual Disk ReadLatency
expr: rate(node_disk_read_time_seconds_total[30m]) / rate(node_disk_reads_completed_total[30m]) > 0.3 and rate(node_disk_reads_completed_total[30m]) > 0
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk read latency {{ $labels.instance }} - {{ $value }} s
|
ok
|
|
26.523s ago
|
18.56ms |
| alert: Node Unusual Disk WriteLatency
expr: rate(node_disk_write_time_seconds_total[30m]) / rate(node_disk_writes_completed_total[30m]) > 0.15 and rate(node_disk_writes_completed_total[30m]) > 0
for: 30m
labels:
severity: critical
annotations:
summary: Host unusual disk write latency {{ $labels.instance }} - {{ $value }} s
|
ok
|
|
26.505s ago
|
18.46ms |
| alert: Big IOWait
expr: avg by(instance) (irate(node_cpu_seconds_total{mode="iowait"}[1m])) * 100 > 15
for: 2m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} have big iowait query: {{ $value }}%'
|
ok
|
|
26.487s ago
|
9.794ms |
| alert: Node High Cpu Load
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{job="node",mode="idle"}[15m])) * 100) > 80
for: 15m
labels:
severity: critical
annotations:
summary: Host high CPU load {{ $labels.instance }} - {{ $value }}%
|
ok
|
|
26.477s ago
|
10.46ms |
| alert: Node Physical Component TooHot
expr: node_hwmon_temp_celsius{chip!~"nvme.+"} > 78
for: 10m
labels:
severity: critical
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }}) {{ $labels.chip }} - {{ $value }}C
|
ok
|
|
26.467s ago
|
4.561ms |
| alert: Node NVMe Drive TooHot
expr: node_hwmon_temp_celsius{chip=~"nvme.+"} > 100
for: 10m
labels:
severity: critical
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }}) {{ $labels.chip }} - {{ $value }}C
|
ok
|
|
26.462s ago
|
629.7us |
| alert: Node Overtemperature Alarm
expr: node_hwmon_temp_crit_alarm_celsius == 1
labels:
severity: critical
annotations:
summary: Host node overtemperature alarm (instance {{ $labels.instance }}) - {{ $value }}
|
ok
|
|
26.462s ago
|
5.663ms |
| alert: Node Raid Array Got Inactive
expr: node_md_state{state="inactive"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically
|
ok
|
|
26.456s ago
|
593.9us |
| alert: Node Raid Disk Failure
expr: node_md_disks{state="failed"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap
|
ok
|
|
26.456s ago
|
529.1us |
| alert: Node Edac Correctable Errors Detected
expr: increase(node_edac_correctable_errors_total[1m]) > 0
for: 10m
labels:
severity: critical
annotations:
summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }}) {{ $value }}
|
ok
|
|
26.456s ago
|
1.23ms |
| alert: Node Edac Uncorrectable Errors Detected
expr: node_edac_uncorrectable_errors_total > 0
labels:
severity: critical
annotations:
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }}) {{ $value }}
|
ok
|
|
26.455s ago
|
1.054ms |
| alert: Relocated Sectors
expr: smart_attribute_raw{attribute_id=~"197|198"} > 0
for: 3h
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} имеет {{ $value }} переназначенных секторов. {{ $labels.device_model_family }} {{ $labels.device_model_name }} под именем {{ $labels.device_name }} с серийным номером {{ $labels.device_serial_number }}'
|
ok
|
|
26.454s ago
|
4.652ms |
| alert: Node Clock Not Synchronising
expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
for: 2m
labels:
severity: critical
annotations:
summary: Clock not synchronising. Ensure NTP is configured on this host {{ $labels.instance }}
|
ok
|
|
26.449s ago
|
1.658ms |
| alert: HostClockSkew
expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
for: 2m
labels:
severity: warning
annotations:
summary: Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host {{ $labels.instance }}
|
ok
|
|
26.448s ago
|
2.421ms |
| alert: Node Disk Is Missing
expr: node_btrfs_device_size_bytes == 0
for: 1m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has a disk {{ $labels.device }} that is not in the file system'
|
ok
|
|
26.446s ago
|
2.43ms |
|
820ms ago |
5.179ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: Prometheus Configuration Failure
expr: prometheus_config_last_reload_successful != 1
labels:
severity: critical
annotations:
summary: Prometheus configuration reload failure (instance {{ $labels.instance }})
|
ok
|
|
820ms ago
|
502.7us |
| alert: AlertManager Configuration Failure
expr: alertmanager_config_last_reload_successful != 1
labels:
severity: critical
annotations:
summary: AlertManager configuration reload failure (instance {{ $labels.iinstance }})
|
ok
|
|
820ms ago
|
275us |
| alert: Prometheus NotConnected To Alertmanager
expr: prometheus_notifications_alertmanagers_discovered < 1
labels:
severity: critical
annotations:
summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }})
|
ok
|
|
820ms ago
|
209.9us |
| alert: Prometheus Rule Evaluation Failures
expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
labels:
severity: critical
annotations:
summary: Prometheus rule evaluation failures (instance {{ $labels.instance }})
|
ok
|
|
820ms ago
|
547.4us |
| alert: Prometheus Template Text Expansion Failures
expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0
labels:
severity: critical
annotations:
summary: Prometheus template text expansion failures (instance {{ $labels.instance }})
|
ok
|
|
819ms ago
|
284.5us |
| alert: Prometheus Rule Evaluation Slow
expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds
for: 5m
labels:
severity: critical
annotations:
summary: Prometheus rule evaluation slow (instance {{ $labels.instance }})
|
ok
|
|
819ms ago
|
775.2us |
| alert: Prometheus Target Empty
expr: prometheus_sd_discovered_targets == 0
labels:
severity: critical
annotations:
description: Prometheus has no target in service discovery
summary: Prometheus target empty (instance {{ $labels.instance }})
|
ok
|
|
819ms ago
|
1.854ms |
| alert: Prometheus Large Scrape
expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10
for: 5m
labels:
severity: critical
annotations:
description: Prometheus has many scrapes that exceed the sample limit
summary: Prometheus large scrape (instance {{ $labels.instance }})
|
ok
|
|
817ms ago
|
421.9us |
| alert: Prometheus Target Scrape Duplicate
expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0
labels:
severity: critical
annotations:
description: Prometheus has many samples rejected due to duplicate timestamps but different values
summary: Prometheus target scrape duplicate (instance {{ $labels.instance }})
|
ok
|
|
817ms ago
|
272.3us |
|
20.746s ago |
8.555ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: UPS not response
expr: up{job=~"apcsnmp|upssnmp"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: UPS {{ $labels.instance }} not responding for more than 5 minutes.
|
ok
|
|
20.746s ago
|
2.123ms |
| alert: Server Room In FIRE!
expr: iemStatusProbeCurrentTemp{instance="Symmetra.ups"} > 24
for: 10m
labels:
severity: critical
annotations:
summary: 'Server room too hot: {{ $value }}C'
|
ok
|
|
20.744s ago
|
410.6us |
| alert: There is no ELECTRICITY in the Server Room!
expr: upsBasicInputPhase{instance="Symmetra.ups"} < 3
for: 5m
labels:
severity: critical
annotations:
summary: 'Working phases in Symmetra at the moment: {{ $value }}'
|
ok
|
|
20.744s ago
|
331.7us |
| alert: APC UPS too hot!
expr: iemStatusProbeCurrentTemp > 30
for: 5m
labels:
severity: critical
annotations:
summary: Too hot in {{ $labels.instance }}. Temperatur is {{ $value }}C
|
ok
|
|
20.744s ago
|
255.4us |
| alert: UPS too hot!
expr: upsBatteryTemperature > 45
for: 5m
labels:
severity: critical
annotations:
summary: Too hot in {{ $labels.instance }}. Temperatur is {{ $value }}C
|
ok
|
|
20.744s ago
|
1.437ms |
| alert: APC UPS low Input voltage
expr: upsAdvInputLineVoltage < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'APC UPS {{ $labels.instance }} has low voltage: {{ $value }}V'
|
ok
|
|
20.743s ago
|
305.2us |
| alert: UPS low Input voltage
expr: upsInputVoltage < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has low input voltage: {{ $value }}V'
|
ok
|
|
20.743s ago
|
846.4us |
| alert: UPS low Output voltage
expr: upsOutputVoltage{upsOutputLineIndex="1"} < 200
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has low output voltage: {{ $value }}V'
|
ok
|
|
20.742s ago
|
727.9us |
| alert: UPS big load
expr: upsOutputPercentLoad{upsOutputLineIndex="1"} > 75
for: 5m
labels:
severity: critical
annotations:
summary: 'UPS {{ $labels.instance }} has big load: {{ $value }}%'
|
ok
|
|
20.741s ago
|
735us |
| alert: Low APC UPS battery capacity
expr: upsAdvBatteryCapacity < 75
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has low battery capacity: {{ $value }}%'
|
ok
|
|
20.741s ago
|
207.3us |
| alert: Low UPS battery capacity
expr: upsEstimatedChargeRemaining < 75
for: 5m
labels:
severity: critical
annotations:
summary: '{{ $labels.instance }} has low battery capacity: {{ $value }}%'
|
ok
|
|
20.741s ago
|
1.13ms |
|
25.944s ago |
50.39ms |