Add dashboards and alerts as code, mark Phase 6 done

Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK) writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service and a system overview -- shipped through a file provider and configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by deleting Grafana and its volume and re-applying. Adds tracesToMetrics to the Tempo datasource. Alerts: Prometheus rule files delivering to Alertmanager, under a labels/annotations contract enforced by observability/alerts/check_contract.py, with runbooks in docs/runbooks/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 20:01:52 +03:30
parent b50f8c3b24
commit fef735c922
19 changed files with 1314 additions and 7 deletions
@@ -0,0 +1,140 @@
+{
+  "annotations": {},
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
+          "legendFormat": "{{http_route}}",
+          "refId": "rate"
+        }
+      ],
+      "title": "Request rate",
+      "transparent": false,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0,
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
+          "legendFormat": "5xx ratio",
+          "refId": "errors"
+        }
+      ],
+      "title": "Error ratio",
+      "transparent": false,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
+          "legendFormat": "p50",
+          "refId": "p50"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
+          "legendFormat": "p95",
+          "refId": "p95"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
+          "legendFormat": "p99",
+          "refId": "p99"
+        }
+      ],
+      "title": "Request duration",
+      "transparent": false,
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "15s",
+  "schemaVersion": 41,
+  "tags": [
+    "generated",
+    "red"
+  ],
+  "templating": {},
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "RED: insights",
+  "uid": "red-insights"
+}