62 lines
3.1 KiB
YAML
62 lines
3.1 KiB
YAML
 
|
# Ledgerline alerting rules.
|
||
|
|
#
|
||
|
|
# ALERTING CONTRACT -- every alert in this directory (except scratch.*) must carry:
|
||
|
|
#
|
||
|
|
# labels (machine-readable; Alertmanager routes and groups on these)
|
||
|
|
# severity critical | warning | info
|
||
|
|
# critical = a human is needed now; warning = look today.
|
||
|
|
# team who owns responding: platform | payments | risk | data
|
||
|
|
# service which Ledgerline service, taken from the series' own label
|
||
|
|
# (aggregations must keep it: `by (service)`)
|
||
|
|
#
|
||
|
|
# annotations (text for humans; never used for routing)
|
||
|
|
# summary one line, no templating beyond the service name
|
||
|
|
# description what is wrong, with the measured value and the threshold
|
||
|
|
# runbook_url what to do about it (docs/runbooks/<kebab-alert-name>.md)
|
||
|
|
# dashboard where to look first (the dashboard that explains the alert)
|
||
|
|
#
|
||
|
|
# observability/alerts/check_contract.py enforces this; run it before applying.
|
||
|
|
#
|
||
|
|
# `team` is static per alert rule, not per service: it says who answers this
|
||
|
|
# *class* of alert. Per-service ownership would need a join against an
|
||
|
|
# ownership metric or one rule per team.
|
||
|
|
groups:
|
||
|
|
- name: ledgerline.rules
|
||
|
|
rules:
|
||
|
|
# All seven targets are scraped through one Traefik address, so a proxy
|
||
|
|
# failure takes every target down at once and fires seven alerts.
|
||
|
|
- alert: LedgerlineTargetDown
|
||
|
|
expr: up{job="ledgerline"} == 0
|
||
|
|
for: 2m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
team: platform
|
||
|
|
annotations:
|
||
|
|
summary: "{{ $labels.service }} is not being scraped"
|
||
|
|
description: "Prometheus has failed to scrape {{ $labels.service }} for more than two minutes."
|
||
|
|
runbook_url: "https://registry.mohsen-hassani.com/mohsen_hassani/LedgerLine/src/branch/main/docs/runbooks/ledgerline-target-down.md"
|
||
|
|
dashboard: "https://grafana.vione.cloud/d/overview"
|
||
|
|
|
||
|
|
# 5xx only (a 404 is the client's mistake). The second condition keeps a
|
||
|
|
# service with almost no traffic from alerting on one failed request:
|
||
|
|
# below 0.05 req/s (about 15 requests per 5m) the ratio is noise. When
|
||
|
|
# there is no traffic at all the ratio is NaN and nothing fires.
|
||
|
|
- alert: LedgerlineHighErrorRatio
|
||
|
|
expr: |
|
||
|
|
(
|
||
|
|
sum by (service) (rate(http_server_request_duration_seconds_count{job="ledgerline",http_response_status_code=~"5.."}[5m]))
|
||
|
|
/
|
||
|
|
sum by (service) (rate(http_server_request_duration_seconds_count{job="ledgerline"}[5m]))
|
||
|
|
) > 0.05
|
||
|
|
and
|
||
|
|
sum by (service) (rate(http_server_request_duration_seconds_count{job="ledgerline"}[5m])) > 0.05
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
team: payments
|
||
|
|
annotations:
|
||
|
|
summary: "{{ $labels.service }} is returning 5xx errors"
|
||
|
|
description: "{{ $labels.service }} has had a 5xx ratio of {{ $value | humanizePercentage }} for five minutes (threshold 5%)."
|
||
|
|
runbook_url: "https://registry.mohsen-hassani.com/mohsen_hassani/LedgerLine/src/branch/main/docs/runbooks/ledgerline-high-error-ratio.md"
|
||
|
|
dashboard: "https://grafana.vione.cloud/d/red-{{ $labels.service }}"
|