Add dashboards and alerts as code, mark Phase 6 done

Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK)
writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service
and a system overview -- shipped through a file provider and
configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by
deleting Grafana and its volume and re-applying. Adds tracesToMetrics to
the Tempo datasource.

Alerts: Prometheus rule files delivering to Alertmanager, under a
labels/annotations contract enforced by observability/alerts/check_contract.py,
with runbooks in docs/runbooks/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

observability/alerts/check_contract.py

@@ -0,0 +1,58 @@
+"""Check every alert rule against the alerting contract (see the header of
+k8s/prometheus/rules/ledgerline.rules.yml).
+
+    python observability/alerts/check_contract.py
+
+Needs PyYAML. Skips scratch.* and host.* rule files.
+"""
+import sys
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+RULES = ROOT / "k8s" / "prometheus" / "rules"
+
+SEVERITIES = {"critical", "warning", "info"}
+TEAMS = {"platform", "payments", "risk", "data"}  # placeholders: edit to taste
+ANNOTATIONS = ("summary", "description", "runbook_url", "dashboard")
+RUNBOOK_MARKER = "/docs/runbooks/"
+
+
+def check(path: Path) -> list[str]:
+    problems = []
+    for group in yaml.safe_load(path.read_text())["groups"]:
+        for rule in group["rules"]:
+            name = rule.get("alert")
+            if name is None:  # recording rule
+                continue
+            where = f"{path.name}:{name}"
+            labels, ann = rule.get("labels", {}), rule.get("annotations", {})
+            if labels.get("severity") not in SEVERITIES:
+                problems.append(f"{where}: severity must be one of {sorted(SEVERITIES)}")
+            if labels.get("team") not in TEAMS:
+                problems.append(f"{where}: team must be one of {sorted(TEAMS)}")
+            for key in ANNOTATIONS:
+                if not ann.get(key):
+                    problems.append(f"{where}: missing annotation {key}")
+            url = ann.get("runbook_url", "")
+            if RUNBOOK_MARKER in url:
+                local = ROOT / "docs" / "runbooks" / url.rsplit("/", 1)[-1]
+                if not local.exists():
+                    problems.append(f"{where}: runbook {local.relative_to(ROOT)} does not exist")
+            elif url:
+                problems.append(f"{where}: runbook_url should point under {RUNBOOK_MARKER}")
+    return problems
+
+
+def main() -> int:
+    files = [p for p in sorted(RULES.glob("*.yml")) if p.name.split(".")[0] not in ("scratch", "host")]
+    problems = [msg for p in files for msg in check(p)]
+    for msg in problems:
+        print(msg)
+    print(f"{len(files)} file(s) checked, {len(problems)} problem(s)")
+    return 1 if problems else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())