Movses AslanyanandClaude Sonnet 5.5
fef735c922
Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK) writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service and a system overview -- shipped through a file provider and configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by deleting Grafana and its volume and re-applying. Adds tracesToMetrics to the Tempo datasource. Alerts: Prometheus rule files delivering to Alertmanager, under a labels/annotations contract enforced by observability/alerts/check_contract.py, with runbooks in docs/runbooks/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
59 lines
2.2 KiB
Python
59 lines
2.2 KiB
Python
"""Check every alert rule against the alerting contract (see the header of
|
|
k8s/prometheus/rules/ledgerline.rules.yml).
|
|
|
|
python observability/alerts/check_contract.py
|
|
|
|
Needs PyYAML. Skips scratch.* and host.* rule files.
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RULES = ROOT / "k8s" / "prometheus" / "rules"
|
|
|
|
SEVERITIES = {"critical", "warning", "info"}
|
|
TEAMS = {"platform", "payments", "risk", "data"} # placeholders: edit to taste
|
|
ANNOTATIONS = ("summary", "description", "runbook_url", "dashboard")
|
|
RUNBOOK_MARKER = "/docs/runbooks/"
|
|
|
|
|
|
def check(path: Path) -> list[str]:
|
|
problems = []
|
|
for group in yaml.safe_load(path.read_text())["groups"]:
|
|
for rule in group["rules"]:
|
|
name = rule.get("alert")
|
|
if name is None: # recording rule
|
|
continue
|
|
where = f"{path.name}:{name}"
|
|
labels, ann = rule.get("labels", {}), rule.get("annotations", {})
|
|
if labels.get("severity") not in SEVERITIES:
|
|
problems.append(f"{where}: severity must be one of {sorted(SEVERITIES)}")
|
|
if labels.get("team") not in TEAMS:
|
|
problems.append(f"{where}: team must be one of {sorted(TEAMS)}")
|
|
for key in ANNOTATIONS:
|
|
if not ann.get(key):
|
|
problems.append(f"{where}: missing annotation {key}")
|
|
url = ann.get("runbook_url", "")
|
|
if RUNBOOK_MARKER in url:
|
|
local = ROOT / "docs" / "runbooks" / url.rsplit("/", 1)[-1]
|
|
if not local.exists():
|
|
problems.append(f"{where}: runbook {local.relative_to(ROOT)} does not exist")
|
|
elif url:
|
|
problems.append(f"{where}: runbook_url should point under {RUNBOOK_MARKER}")
|
|
return problems
|
|
|
|
|
|
def main() -> int:
|
|
files = [p for p in sorted(RULES.glob("*.yml")) if p.name.split(".")[0] not in ("scratch", "host")]
|
|
problems = [msg for p in files for msg in check(p)]
|
|
for msg in problems:
|
|
print(msg)
|
|
print(f"{len(files)} file(s) checked, {len(problems)} problem(s)")
|
|
return 1 if problems else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|