Add dashboards and alerts as code, mark Phase 6 done
Dashboards: observability/dashboards/generate.py (Grafana Foundation SDK) writes k8s/grafana/dashboards/*.json -- a RED dashboard per HTTP service and a system overview -- shipped through a file provider and configMapGenerators, so `kubectl apply -k k8s/` rebuilds them. Verified by deleting Grafana and its volume and re-applying. Adds tracesToMetrics to the Tempo datasource. Alerts: Prometheus rule files delivering to Alertmanager, under a labels/annotations contract enforced by observability/alerts/check_contract.py, with runbooks in docs/runbooks/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Movses Aslanyan
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
b50f8c3b24
commit
fef735c922
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: ledgerline
|
||||
orgId: 1
|
||||
type: file
|
||||
folder: Ledgerline
|
||||
options:
|
||||
path: /etc/grafana/dashboards/
|
||||
allowUiUpdates: false
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 15
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"filters": [],
|
||||
"queryType": "serviceMap",
|
||||
"refId": "graph"
|
||||
}
|
||||
],
|
||||
"title": "Service graph",
|
||||
"transparent": false,
|
||||
"type": "nodeGraph"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "ledgerline_stream_lag",
|
||||
"legendFormat": "{{consumer_group}}",
|
||||
"refId": "lag"
|
||||
}
|
||||
],
|
||||
"title": "Stream consumer lag",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (service, pool) (ledgerline_db_pool_in_use)",
|
||||
"legendFormat": "{{service}} / {{pool}}",
|
||||
"refId": "pool"
|
||||
}
|
||||
],
|
||||
"title": "DB pool connections in use",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"overview"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "System overview",
|
||||
"uid": "overview"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
|
||||
"legendFormat": "{{http_route}}",
|
||||
"refId": "rate"
|
||||
}
|
||||
],
|
||||
"title": "Request rate",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"min": 0,
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"fraud\"}[5m]))",
|
||||
"legendFormat": "5xx ratio",
|
||||
"refId": "errors"
|
||||
}
|
||||
],
|
||||
"title": "Error ratio",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"fraud\"}[5m])))",
|
||||
"legendFormat": "p99",
|
||||
"refId": "p99"
|
||||
}
|
||||
],
|
||||
"title": "Request duration",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"red"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "RED: fraud",
|
||||
"uid": "red-fraud"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\"}[5m]))",
|
||||
"legendFormat": "{{http_route}}",
|
||||
"refId": "rate"
|
||||
}
|
||||
],
|
||||
"title": "Request rate",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"min": 0,
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"gateway\"}[5m]))",
|
||||
"legendFormat": "5xx ratio",
|
||||
"refId": "errors"
|
||||
}
|
||||
],
|
||||
"title": "Error ratio",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"gateway\"}[5m])))",
|
||||
"legendFormat": "p99",
|
||||
"refId": "p99"
|
||||
}
|
||||
],
|
||||
"title": "Request duration",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"red"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "RED: gateway",
|
||||
"uid": "red-gateway"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
|
||||
"legendFormat": "{{http_route}}",
|
||||
"refId": "rate"
|
||||
}
|
||||
],
|
||||
"title": "Request rate",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"min": 0,
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"insights\"}[5m]))",
|
||||
"legendFormat": "5xx ratio",
|
||||
"refId": "errors"
|
||||
}
|
||||
],
|
||||
"title": "Error ratio",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"insights\"}[5m])))",
|
||||
"legendFormat": "p99",
|
||||
"refId": "p99"
|
||||
}
|
||||
],
|
||||
"title": "Request duration",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"red"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "RED: insights",
|
||||
"uid": "red-insights"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\"}[5m]))",
|
||||
"legendFormat": "{{http_route}}",
|
||||
"refId": "rate"
|
||||
}
|
||||
],
|
||||
"title": "Request rate",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"min": 0,
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"ledger\"}[5m]))",
|
||||
"legendFormat": "5xx ratio",
|
||||
"refId": "errors"
|
||||
}
|
||||
],
|
||||
"title": "Error ratio",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"ledger\"}[5m])))",
|
||||
"legendFormat": "p99",
|
||||
"refId": "p99"
|
||||
}
|
||||
],
|
||||
"title": "Request duration",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"red"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "RED: ledger",
|
||||
"uid": "red-ledger"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"annotations": {},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\"}[5m]))",
|
||||
"legendFormat": "{{http_route}}",
|
||||
"refId": "rate"
|
||||
}
|
||||
],
|
||||
"title": "Request rate",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"min": 0,
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\",http_response_status_code=~\"5..\"}[5m])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"ledgerline\",service=\"sanctions\"}[5m]))",
|
||||
"legendFormat": "5xx ratio",
|
||||
"refId": "errors"
|
||||
}
|
||||
],
|
||||
"title": "Error ratio",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"repeatDirection": "h",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
|
||||
"legendFormat": "p50",
|
||||
"refId": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
|
||||
"legendFormat": "p95",
|
||||
"refId": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"ledgerline\",service=\"sanctions\"}[5m])))",
|
||||
"legendFormat": "p99",
|
||||
"refId": "p99"
|
||||
}
|
||||
],
|
||||
"title": "Request duration",
|
||||
"transparent": false,
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": [
|
||||
"generated",
|
||||
"red"
|
||||
],
|
||||
"templating": {},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "RED: sanctions",
|
||||
"uid": "red-sanctions"
|
||||
}
|
||||
@@ -38,6 +38,22 @@ datasources:
|
||||
filterByTraceID: true
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
# QUESTION (answer in your own words; verify by reading the LogQL that
|
||||
# "Logs for this span" generates): why is `filterByTraceID: true` enough
|
||||
# with no `tags` mapping here, and when would it not be?
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
# Wider than the span: rate() needs a few scrape intervals of data.
|
||||
spanStartTimeShift: -2m
|
||||
spanEndTimeShift: 2m
|
||||
# span attribute -> Prometheus label on traces_spanmetrics_*
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
queries:
|
||||
# `$$` is the provisioning-file escape for a literal `$`.
|
||||
- name: Request rate
|
||||
query: sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
|
||||
@@ -51,6 +51,10 @@ spec:
|
||||
mountPath: /var/lib/grafana
|
||||
- name: datasources
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
- name: dashboards-provider
|
||||
mountPath: /etc/grafana/provisioning/dashboards
|
||||
- name: dashboards
|
||||
mountPath: /etc/grafana/dashboards/
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
@@ -58,3 +62,9 @@ spec:
|
||||
- name: datasources
|
||||
configMap:
|
||||
name: grafana-datasources
|
||||
- name: dashboards-provider
|
||||
configMap:
|
||||
name: grafana-dashboards-provider
|
||||
- name: dashboards
|
||||
configMap:
|
||||
name: grafana-dashboards
|
||||
|
||||
@@ -7,9 +7,18 @@ resources:
|
||||
- service.yaml
|
||||
- ingress.yaml
|
||||
|
||||
# Datasources only, as in the Ansible role. Dashboard provisioning is the
|
||||
# Phase 6 deliverable and belongs in this directory.
|
||||
configMapGenerator:
|
||||
- name: grafana-datasources
|
||||
files:
|
||||
- datasources.yml
|
||||
- name: grafana-dashboards-provider
|
||||
files:
|
||||
- dashboards-provider.yml
|
||||
- name: grafana-dashboards
|
||||
files:
|
||||
- dashboards/overview.json
|
||||
- dashboards/red-fraud.json
|
||||
- dashboards/red-gateway.json
|
||||
- dashboards/red-ledger.json
|
||||
- dashboards/red-sanctions.json
|
||||
- dashboards/red-insights.json
|
||||
|
||||
Reference in New Issue
Block a user