mohsen_hassani/LedgerLine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dashboards and alerts as code, mark Phase 6 done

Browse Source 
Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK)
writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service
and a system overview -- shipped through a file provider and
configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by
deleting Grafana and its volume and re-applying. Adds tracesToMetrics to
the Tempo datasource.

Alerts: Prometheus rule files delivering to Alertmanager, under a
labels/annotations contract enforced by observability/alerts/check_contract.py,
with runbooks in docs/runbooks/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Movses Aslanyan
2026-10-25 20:01:52 +03:30
co-authored by Claude Sonnet 5.5
parent b50f8c3b24
commit fef735c922
19 changed files with 1314 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/grafana/dashboards/overview.json
+115
View File
@@ -0,0 +1,115 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"filters": [],
"queryType": "serviceMap",
"refId": "graph"
}
],
"title": "Service graph",
"transparent": false,
"type": "nodeGraph"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 12
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "ledgerline_stream_lag",
"legendFormat": "{{consumer_group}}",
"refId": "lag"
}
],
"title": "Stream consumer lag",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 12
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (service, pool) (ledgerline_db_pool_in_use)",
"legendFormat": "{{service}} / {{pool}}",
"refId": "pool"
}
],
"title": "DB pool connections in use",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"overview"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "System overview",
"uid": "overview"
}
k8s/grafana/dashboards/red-fraud.json
+140
View File
@@ -0,0 +1,140 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
"legendFormat": "{{http_route}}",
"refId": "rate"
}
],
"title": "Request rate",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"min": 0,
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
"legendFormat": "5xx ratio",
"refId": "errors"
}
],
"title": "Error ratio",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
"legendFormat": "p50",
"refId": "p50"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
"legendFormat": "p95",
"refId": "p95"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
"legendFormat": "p99",
"refId": "p99"
}
],
"title": "Request duration",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"red"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "RED: fraud",
"uid": "red-fraud"
}
k8s/grafana/dashboards/red-gateway.json
+140
View File
@@ -0,0 +1,140 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\"}[5m]))",
"legendFormat": "{{http_route}}",
"refId": "rate"
}
],
"title": "Request rate",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"min": 0,
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\"}[5m]))",
"legendFormat": "5xx ratio",
"refId": "errors"
}
],
"title": "Error ratio",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
"legendFormat": "p50",
"refId": "p50"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
"legendFormat": "p95",
"refId": "p95"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
"legendFormat": "p99",
"refId": "p99"
}
],
"title": "Request duration",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"red"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "RED: gateway",
"uid": "red-gateway"
}
k8s/grafana/dashboards/red-insights.json
+140
View File
@@ -0,0 +1,140 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
"legendFormat": "{{http_route}}",
"refId": "rate"
}
],
"title": "Request rate",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"min": 0,
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
"legendFormat": "5xx ratio",
"refId": "errors"
}
],
"title": "Error ratio",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
"legendFormat": "p50",
"refId": "p50"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
"legendFormat": "p95",
"refId": "p95"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
"legendFormat": "p99",
"refId": "p99"
}
],
"title": "Request duration",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"red"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "RED: insights",
"uid": "red-insights"
}
k8s/grafana/dashboards/red-ledger.json
+140
View File
@@ -0,0 +1,140 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\"}[5m]))",
"legendFormat": "{{http_route}}",
"refId": "rate"
}
],
"title": "Request rate",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"min": 0,
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\"}[5m]))",
"legendFormat": "5xx ratio",
"refId": "errors"
}
],
"title": "Error ratio",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
"legendFormat": "p50",
"refId": "p50"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
"legendFormat": "p95",
"refId": "p95"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
"legendFormat": "p99",
"refId": "p99"
}
],
"title": "Request duration",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"red"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "RED: ledger",
"uid": "red-ledger"
}
k8s/grafana/dashboards/red-sanctions.json
+140
View File
@@ -0,0 +1,140 @@
{
"annotations": {},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\"}[5m]))",
"legendFormat": "{{http_route}}",
"refId": "rate"
}
],
"title": "Request rate",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"min": 0,
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\"}[5m]))",
"legendFormat": "5xx ratio",
"refId": "errors"
}
],
"title": "Error ratio",
"transparent": false,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 9
},
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
"legendFormat": "p50",
"refId": "p50"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
"legendFormat": "p95",
"refId": "p95"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
"legendFormat": "p99",
"refId": "p99"
}
],
"title": "Request duration",
"transparent": false,
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 41,
"tags": [
"generated",
"red"
],
"templating": {},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"title": "RED: sanctions",
"uid": "red-sanctions"
}