Add dashboards and alerts as code, mark Phase 6 done
Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK) writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service and a system overview -- shipped through a file provider and configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by deleting Grafana and its volume and re-applying. Adds tracesToMetrics to the Tempo datasource. Alerts: Prometheus rule files delivering to Alertmanager, under a labels/annotations contract enforced by observability/alerts/check_contract.py, with runbooks in docs/runbooks/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Movses Aslanyan
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
b50f8c3b24
commit
fef735c922
@@ -0,0 +1,58 @@
|
||||
"""Check every alert rule against the alerting contract (see the header of
|
||||
k8s/prometheus/rules/ledgerline.rules.yml).
|
||||
|
||||
python observability/alerts/check_contract.py
|
||||
|
||||
Needs PyYAML. Skips scratch.* and host.* rule files.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RULES = ROOT / "k8s" / "prometheus" / "rules"
|
||||
|
||||
SEVERITIES = {"critical", "warning", "info"}
|
||||
TEAMS = {"platform", "payments", "risk", "data"} # placeholders: edit to taste
|
||||
ANNOTATIONS = ("summary", "description", "runbook_url", "dashboard")
|
||||
RUNBOOK_MARKER = "/docs/runbooks/"
|
||||
|
||||
|
||||
def check(path: Path) -> list[str]:
|
||||
problems = []
|
||||
for group in yaml.safe_load(path.read_text())["groups"]:
|
||||
for rule in group["rules"]:
|
||||
name = rule.get("alert")
|
||||
if name is None: # recording rule
|
||||
continue
|
||||
where = f"{path.name}:{name}"
|
||||
labels, ann = rule.get("labels", {}), rule.get("annotations", {})
|
||||
if labels.get("severity") not in SEVERITIES:
|
||||
problems.append(f"{where}: severity must be one of {sorted(SEVERITIES)}")
|
||||
if labels.get("team") not in TEAMS:
|
||||
problems.append(f"{where}: team must be one of {sorted(TEAMS)}")
|
||||
for key in ANNOTATIONS:
|
||||
if not ann.get(key):
|
||||
problems.append(f"{where}: missing annotation {key}")
|
||||
url = ann.get("runbook_url", "")
|
||||
if RUNBOOK_MARKER in url:
|
||||
local = ROOT / "docs" / "runbooks" / url.rsplit("/", 1)[-1]
|
||||
if not local.exists():
|
||||
problems.append(f"{where}: runbook {local.relative_to(ROOT)} does not exist")
|
||||
elif url:
|
||||
problems.append(f"{where}: runbook_url should point under {RUNBOOK_MARKER}")
|
||||
return problems
|
||||
|
||||
|
||||
def main() -> int:
|
||||
files = [p for p in sorted(RULES.glob("*.yml")) if p.name.split(".")[0] not in ("scratch", "host")]
|
||||
problems = [msg for p in files for msg in check(p)]
|
||||
for msg in problems:
|
||||
print(msg)
|
||||
print(f"{len(files)} file(s) checked, {len(problems)} problem(s)")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Generate Grafana dashboard JSON into k8s/grafana/dashboards/.
|
||||
|
||||
python -m venv .venv && .venv/bin/pip install -r requirements.txt
|
||||
.venv/bin/python generate.py
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from grafana_foundation_sdk.builders.dashboard import Dashboard
|
||||
from grafana_foundation_sdk.builders.prometheus import Dataquery as PromQuery
|
||||
from grafana_foundation_sdk.builders.nodegraph import Panel as NodeGraph
|
||||
from grafana_foundation_sdk.builders.tempo import TempoQuery
|
||||
from grafana_foundation_sdk.builders.timeseries import Panel as Timeseries
|
||||
from grafana_foundation_sdk.cog.encoder import JSONEncoder
|
||||
from grafana_foundation_sdk.models.dashboard import DataSourceRef
|
||||
|
||||
OUT_DIR = Path(__file__).resolve().parents[2] / "k8s" / "grafana" / "dashboards"
|
||||
|
||||
PROMETHEUS = DataSourceRef(type_val="prometheus", uid="prometheus")
|
||||
|
||||
|
||||
# Audit and Notifier are consumers and does not have HTTP, so commented them out for now
|
||||
SERVICES = [
|
||||
"gateway",
|
||||
"ledger",
|
||||
"fraud",
|
||||
"sanctions",
|
||||
"insights",
|
||||
# "audit",
|
||||
# "notifier",
|
||||
]
|
||||
|
||||
|
||||
def prom(expr: str, ref_id: str, legend: str) -> PromQuery:
|
||||
return (
|
||||
PromQuery()
|
||||
.expr(expr)
|
||||
.ref_id(ref_id)
|
||||
.legend_format(legend)
|
||||
.datasource(PROMETHEUS)
|
||||
)
|
||||
|
||||
|
||||
def red_dashboard(service: str) -> Dashboard:
|
||||
"""Rate, Errors, Duration for one service, in the order you'd ask them
|
||||
during an incident: is it serving, is it failing, is it slow."""
|
||||
# `service` and `instance` carry the same value on these series.
|
||||
sel = f'job="ledgerline",service="{service}"'
|
||||
count = f"http_server_request_duration_seconds_count{{{sel}}}"
|
||||
bucket = f"http_server_request_duration_seconds_bucket{{{sel}}}"
|
||||
|
||||
def quantile(q: float) -> str:
|
||||
return f"histogram_quantile({q}, sum by (le) (rate({bucket}[5m])))"
|
||||
|
||||
rate = (
|
||||
Timeseries()
|
||||
.title("Request rate")
|
||||
.datasource(PROMETHEUS)
|
||||
.unit("reqps")
|
||||
.with_target(
|
||||
prom(
|
||||
f"sum by (http_route) (rate({count}[5m]))",
|
||||
"rate",
|
||||
"{{http_route}}",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
errors = (
|
||||
Timeseries()
|
||||
.title("Error ratio")
|
||||
.datasource(PROMETHEUS)
|
||||
.unit("percentunit")
|
||||
.min(0)
|
||||
.with_target(
|
||||
prom(
|
||||
# 5xx only: a 404 is the client's mistake, not the service failing.
|
||||
# `or vector(0)` keeps the line at 0 when no 5xx series exists yet;
|
||||
# with no traffic the denominator is 0 and the result is NaN (a gap).
|
||||
f'(sum(rate({count[:-1]},http_response_status_code=~"5.."}}[5m])) or vector(0))'
|
||||
f" / sum(rate({count}[5m]))",
|
||||
"errors",
|
||||
"5xx ratio",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
duration = (
|
||||
Timeseries()
|
||||
.title("Request duration")
|
||||
.datasource(PROMETHEUS)
|
||||
.unit("s")
|
||||
# One query per quantile. Aggregate the histogram *buckets* first and
|
||||
# compute the quantile after; never average per-instance quantiles.
|
||||
.with_target(prom(quantile(0.50), "p50", "p50"))
|
||||
.with_target(prom(quantile(0.95), "p95", "p95"))
|
||||
.with_target(prom(quantile(0.99), "p99", "p99"))
|
||||
)
|
||||
|
||||
return (
|
||||
Dashboard(f"RED: {service}")
|
||||
.uid(f"red-{service}")
|
||||
.tags(["generated", "red"])
|
||||
.time("now-1h", "now")
|
||||
.refresh("15s")
|
||||
.readonly()
|
||||
.with_panel(rate)
|
||||
.with_panel(errors)
|
||||
.with_panel(duration)
|
||||
)
|
||||
|
||||
|
||||
TEMPO = DataSourceRef(type_val="tempo", uid="tempo")
|
||||
|
||||
|
||||
def overview_dashboard() -> Dashboard:
|
||||
"""System-wide: who calls whom, is the async hop keeping up, is a pool
|
||||
under pressure. Not per service, so no `service` variable."""
|
||||
|
||||
# Not a backend query: Grafana's browser builds the node graph from the
|
||||
# Prometheus span metrics that Tempo's metrics-generator writes (wired by
|
||||
# `serviceMap.datasourceUid` in datasources.yml). Calling it through
|
||||
# /api/ds/query fails with "unsupported query type", so this panel can only
|
||||
# be checked by looking at it in the browser.
|
||||
service_graph = (
|
||||
NodeGraph()
|
||||
.title("Service graph")
|
||||
.datasource(TEMPO)
|
||||
.span(24)
|
||||
.height(12)
|
||||
.with_target(
|
||||
TempoQuery()
|
||||
.datasource(TEMPO)
|
||||
.ref_id("graph")
|
||||
.query_type("serviceMap")
|
||||
)
|
||||
)
|
||||
|
||||
lag = (
|
||||
Timeseries()
|
||||
.title("Stream consumer lag")
|
||||
.datasource(PROMETHEUS)
|
||||
.span(12)
|
||||
.unit("none")
|
||||
.with_target(
|
||||
prom(
|
||||
"ledgerline_stream_lag",
|
||||
"lag",
|
||||
"{{consumer_group}}",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
pool = (
|
||||
Timeseries()
|
||||
.title("DB pool connections in use")
|
||||
.datasource(PROMETHEUS)
|
||||
.span(12)
|
||||
.unit("none")
|
||||
.with_target(
|
||||
prom(
|
||||
# "In use", not a percentage: the app exports no pool *size*.
|
||||
# What would you need to turn this into saturation, and where
|
||||
# could it come from? (Also look at the labels of
|
||||
# `db_client_connections_usage_connections{state=...}`.)
|
||||
"sum by (service, pool) (ledgerline_db_pool_in_use)",
|
||||
"pool",
|
||||
"{{service}} / {{pool}}",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return (
|
||||
Dashboard("System overview")
|
||||
.uid("overview")
|
||||
.tags(["generated", "overview"])
|
||||
.time("now-1h", "now")
|
||||
.refresh("15s")
|
||||
.readonly()
|
||||
.with_panel(service_graph)
|
||||
.with_panel(lag)
|
||||
.with_panel(pool)
|
||||
)
|
||||
|
||||
|
||||
DASHBOARDS = {
|
||||
"overview": overview_dashboard,
|
||||
**{f"red-{svc}": (lambda s=svc: red_dashboard(s)) for svc in SERVICES},
|
||||
}
|
||||
|
||||
# A ConfigMap is capped at ~1 MiB, and client-side `kubectl apply` stores a
|
||||
# second copy in an annotation, so keep well under half of it.
|
||||
MAX_TOTAL_BYTES = 400_000
|
||||
|
||||
|
||||
def main() -> None:
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
rendered = {
|
||||
name: json.dumps(fn().build(), cls=JSONEncoder, indent=2, sort_keys=True) + "\n"
|
||||
for name, fn in DASHBOARDS.items()
|
||||
}
|
||||
for name, text in rendered.items():
|
||||
assert "FIXME" not in text, f"{name}: unfilled FIXME in generated JSON"
|
||||
total = sum(len(t) for t in rendered.values())
|
||||
assert total < MAX_TOTAL_BYTES, f"{total} bytes of dashboards; ConfigMap limit"
|
||||
for name, text in rendered.items():
|
||||
(OUT_DIR / f"{name}.json").write_text(text)
|
||||
print(f"wrote {OUT_DIR / f'{name}.json'}")
|
||||
print(f"{total} bytes total")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
# Epoch!grafana-version. The unpinned default is the Grafana 10.1 schema (highest
|
||||
# epoch = oldest schema); 11.6.0 is the newest published, loads fine in 12.1.1.
|
||||
grafana-foundation-sdk==1769699379!11.6.0
|
||||
Reference in New Issue
Block a user