Movses AslanyanandClaude Sonnet 5.5
fef735c922
Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK) writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service and a system overview -- shipped through a file provider and configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by deleting Grafana and its volume and re-applying. Adds tracesToMetrics to the Tempo datasource. Alerts: Prometheus rule files delivering to Alertmanager, under a labels/annotations contract enforced by observability/alerts/check_contract.py, with runbooks in docs/runbooks/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
141 lines
3.4 KiB
JSON
141 lines
3.4 KiB
JSON
{
|
|
"annotations": {},
|
|
"editable": false,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"repeatDirection": "h",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
|
|
"legendFormat": "{{http_route}}",
|
|
"refId": "rate"
|
|
}
|
|
],
|
|
"title": "Request rate",
|
|
"transparent": false,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"min": 0,
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"repeatDirection": "h",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
|
|
"legendFormat": "5xx ratio",
|
|
"refId": "errors"
|
|
}
|
|
],
|
|
"title": "Error ratio",
|
|
"transparent": false,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"repeatDirection": "h",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
|
"legendFormat": "p50",
|
|
"refId": "p50"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
|
"legendFormat": "p95",
|
|
"refId": "p95"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
|
"legendFormat": "p99",
|
|
"refId": "p99"
|
|
}
|
|
],
|
|
"title": "Request duration",
|
|
"transparent": false,
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "15s",
|
|
"schemaVersion": 41,
|
|
"tags": [
|
|
"generated",
|
|
"red"
|
|
],
|
|
"templating": {},
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"title": "RED: fraud",
|
|
"uid": "red-fraud"
|
|
}
|