LedgerLine/observability/dashboards/generate.py

"""Generate Grafana dashboard JSON into k8s/grafana/dashboards/.

    python -m venv .venv && .venv/bin/pip install -r requirements.txt
    .venv/bin/python generate.py
"""
import json
from pathlib import Path

from grafana_foundation_sdk.builders.dashboard import Dashboard
from grafana_foundation_sdk.builders.prometheus import Dataquery as PromQuery
from grafana_foundation_sdk.builders.nodegraph import Panel as NodeGraph
from grafana_foundation_sdk.builders.tempo import TempoQuery
from grafana_foundation_sdk.builders.timeseries import Panel as Timeseries
from grafana_foundation_sdk.cog.encoder import JSONEncoder
from grafana_foundation_sdk.models.dashboard import DataSourceRef

OUT_DIR = Path(__file__).resolve().parents[2] / "k8s" / "grafana" / "dashboards"

PROMETHEUS = DataSourceRef(type_val="prometheus", uid="prometheus")


# Audit and Notifier are consumers and does not have HTTP, so commented them out for now
SERVICES = [
    "gateway",
    "ledger",
    "fraud",
    "sanctions",
    "insights",
    # "audit",
    # "notifier",
]


def prom(expr: str, ref_id: str, legend: str) -> PromQuery:
    return (
        PromQuery()
        .expr(expr)
        .ref_id(ref_id)
        .legend_format(legend)
        .datasource(PROMETHEUS)
    )


def red_dashboard(service: str) -> Dashboard:
    """Rate, Errors, Duration for one service, in the order you'd ask them
    during an incident: is it serving, is it failing, is it slow."""
    # `service` and `instance` carry the same value on these series.
    sel = f'job="ledgerline",service="{service}"'
    count = f"http_server_request_duration_seconds_count{{{sel}}}"
    bucket = f"http_server_request_duration_seconds_bucket{{{sel}}}"

    def quantile(q: float) -> str:
        return f"histogram_quantile({q}, sum by (le) (rate({bucket}[5m])))"

    rate = (
        Timeseries()
        .title("Request rate")
        .datasource(PROMETHEUS)
        .unit("reqps")
        .with_target(
            prom(
                f"sum by (http_route) (rate({count}[5m]))",
                "rate",
                "{{http_route}}",
            )
        )
    )

    errors = (
        Timeseries()
        .title("Error ratio")
        .datasource(PROMETHEUS)
        .unit("percentunit")
        .min(0)
        .with_target(
            prom(
                # 5xx only: a 404 is the client's mistake, not the service failing.
                # `or vector(0)` keeps the line at 0 when no 5xx series exists yet;
                # with no traffic the denominator is 0 and the result is NaN (a gap).
                f'(sum(rate({count[:-1]},http_response_status_code=~"5.."}}[5m])) or vector(0))'
                f" / sum(rate({count}[5m]))",
                "errors",
                "5xx ratio",
            )
        )
    )

    duration = (
        Timeseries()
        .title("Request duration")
        .datasource(PROMETHEUS)
        .unit("s")
        # One query per quantile. Aggregate the histogram *buckets* first and
        # compute the quantile after; never average per-instance quantiles.
        .with_target(prom(quantile(0.50), "p50", "p50"))
        .with_target(prom(quantile(0.95), "p95", "p95"))
        .with_target(prom(quantile(0.99), "p99", "p99"))
    )

    return (
        Dashboard(f"RED: {service}")
        .uid(f"red-{service}")
        .tags(["generated", "red"])
        .time("now-1h", "now")
        .refresh("15s")
        .readonly()
        .with_panel(rate)
        .with_panel(errors)
        .with_panel(duration)
    )


TEMPO = DataSourceRef(type_val="tempo", uid="tempo")


def overview_dashboard() -> Dashboard:
    """System-wide: who calls whom, is the async hop keeping up, is a pool
    under pressure. Not per service, so no `service` variable."""

    # Not a backend query: Grafana's browser builds the node graph from the
    # Prometheus span metrics that Tempo's metrics-generator writes (wired by
    # `serviceMap.datasourceUid` in datasources.yml). Calling it through
    # /api/ds/query fails with "unsupported query type", so this panel can only
    # be checked by looking at it in the browser.
    service_graph = (
        NodeGraph()
        .title("Service graph")
        .datasource(TEMPO)
        .span(24)
        .height(12)
        .with_target(
            TempoQuery()
            .datasource(TEMPO)
            .ref_id("graph")
            .query_type("serviceMap")
        )
    )

    lag = (
        Timeseries()
        .title("Stream consumer lag")
        .datasource(PROMETHEUS)
        .span(12)
        .unit("none")
        .with_target(
            prom(
                "ledgerline_stream_lag",
                "lag",
                "{{consumer_group}}",
            )
        )
    )

    pool = (
        Timeseries()
        .title("DB pool connections in use")
        .datasource(PROMETHEUS)
        .span(12)
        .unit("none")
        .with_target(
            prom(
                # "In use", not a percentage: the app exports no pool *size*.
                # What would you need to turn this into saturation, and where
                # could it come from? (Also look at the labels of
                # `db_client_connections_usage_connections{state=...}`.)
                "sum by (service, pool) (ledgerline_db_pool_in_use)",
                "pool",
                "{{service}} / {{pool}}",
            )
        )
    )

    return (
        Dashboard("System overview")
        .uid("overview")
        .tags(["generated", "overview"])
        .time("now-1h", "now")
        .refresh("15s")
        .readonly()
        .with_panel(service_graph)
        .with_panel(lag)
        .with_panel(pool)
    )


DASHBOARDS = {
    "overview": overview_dashboard,
    **{f"red-{svc}": (lambda s=svc: red_dashboard(s)) for svc in SERVICES},
}

# A ConfigMap is capped at ~1 MiB, and client-side `kubectl apply` stores a
# second copy in an annotation, so keep well under half of it.
MAX_TOTAL_BYTES = 400_000


def main() -> None:
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    rendered = {
        name: json.dumps(fn().build(), cls=JSONEncoder, indent=2, sort_keys=True) + "\n"
        for name, fn in DASHBOARDS.items()
    }
    for name, text in rendered.items():
        assert "FIXME" not in text, f"{name}: unfilled FIXME in generated JSON"
    total = sum(len(t) for t in rendered.values())
    assert total < MAX_TOTAL_BYTES, f"{total} bytes of dashboards; ConfigMap limit"
    for name, text in rendered.items():
        (OUT_DIR / f"{name}.json").write_text(text)
        print(f"wrote {OUT_DIR / f'{name}.json'}")
    print(f"{total} bytes total")


if __name__ == "__main__":
    main()