Scrape Ledgerline's metrics from falken through czy's Traefik
czy publishes no service ports, so Traefik exposes each service's :9464 as https://ledgerline.vione.cloud/metrics/<service> behind basic auth, and the platform's Prometheus gets a `ledgerline` job with one target per service. The targets share an address; a `service` label tells them apart, and relabels derive __metrics_path__ and instance from it. create-secrets.sh now also creates the prometheus-scrape-auth secret (mounted optional, so a missing secret fails only this job) and writes the gitignored htpasswd file Traefik reads, which is copied to czy like .env.observability. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Movses Aslanyan
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
920712f32b
commit
b3e7c14958
@@ -4,6 +4,7 @@ __pycache__/
|
||||
.pytest_cache/
|
||||
.env
|
||||
.env.observability
|
||||
observability/traefik/metrics-users
|
||||
k8s/.credentials
|
||||
*.egg-info/
|
||||
.DS_Store
|
||||
|
||||
@@ -67,8 +67,9 @@ at one point in time and the user may have since extended them.
|
||||
2. **App metrics reach Prometheus by direct scrape, not through Alloy.** Alloy has
|
||||
no `prometheus.scrape` component; `k8s/prometheus/prometheus.yml` scrapes the
|
||||
stack's own components, and app jobs get added there (the role's
|
||||
`prometheus_extra_scrape_jobs`). czy's service ports are unpublished, so no app
|
||||
scrape path exists yet — a Phase 5 decision. Note that
|
||||
`prometheus_extra_scrape_jobs`). czy's service ports are unpublished, so the
|
||||
`ledgerline` job scrapes each service's `:9464` through czy's Traefik at
|
||||
`https://ledgerline.vione.cloud/metrics/<service>` (basic auth). Note that
|
||||
the diagram draws this differently (app metrics routed via Alloy) — the diagram
|
||||
and the code have drifted, and the user knows. Don't "correct" one to the other
|
||||
unasked.
|
||||
|
||||
@@ -5,6 +5,13 @@
|
||||
#
|
||||
# Only Traefik publishes host ports (80/443). Everything else is reachable
|
||||
# only on the compose network.
|
||||
#
|
||||
# Metrics: every Ledgerline service serves OTel metrics on :9464 (see
|
||||
# common/telemetry.py). Traefik exposes each one as
|
||||
# https://ledgerline.vione.cloud/metrics/<service>, behind basic auth, so the
|
||||
# platform's Prometheus on falken can scrape it (k8s/prometheus/prometheus.yml,
|
||||
# job `ledgerline`). The htpasswd file is written by k8s/create-secrets.sh and
|
||||
# copied here once, like .env.observability.
|
||||
|
||||
services:
|
||||
traefik:
|
||||
@@ -28,6 +35,7 @@ services:
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- letsencrypt:/letsencrypt
|
||||
- ./observability/traefik/metrics-users:/etc/traefik/metrics-users:ro
|
||||
|
||||
gateway:
|
||||
ports: !reset []
|
||||
@@ -37,6 +45,18 @@ services:
|
||||
- traefik.http.routers.ledgerline.entrypoints=websecure
|
||||
- traefik.http.routers.ledgerline.tls.certresolver=le
|
||||
- traefik.http.services.ledgerline.loadbalancer.server.port=8000
|
||||
# gateway now has two Traefik services, so each router names its own.
|
||||
- traefik.http.routers.ledgerline.service=ledgerline
|
||||
- traefik.http.routers.gateway-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/gateway`)
|
||||
- traefik.http.routers.gateway-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.gateway-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.gateway-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.routers.gateway-metrics.service=gateway-metrics
|
||||
- traefik.http.services.gateway-metrics.loadbalancer.server.port=9464
|
||||
# Shared by every <service>-metrics router. Declared here because a
|
||||
# middleware needs an enabled container to carry it.
|
||||
- traefik.http.middlewares.metrics-auth.basicauth.usersfile=/etc/traefik/metrics-users
|
||||
- traefik.http.middlewares.metrics-path.replacepath.path=/metrics
|
||||
|
||||
postgres:
|
||||
ports: !reset []
|
||||
@@ -46,18 +66,62 @@ services:
|
||||
|
||||
ledger:
|
||||
ports: !reset []
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.ledger-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/ledger`)
|
||||
- traefik.http.routers.ledger-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.ledger-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.ledger-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.ledger-metrics.loadbalancer.server.port=9464
|
||||
|
||||
fraud:
|
||||
ports: !reset []
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.fraud-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/fraud`)
|
||||
- traefik.http.routers.fraud-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.fraud-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.fraud-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.fraud-metrics.loadbalancer.server.port=9464
|
||||
|
||||
sanctions:
|
||||
ports: !reset []
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.sanctions-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/sanctions`)
|
||||
- traefik.http.routers.sanctions-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.sanctions-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.sanctions-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.sanctions-metrics.loadbalancer.server.port=9464
|
||||
|
||||
insights:
|
||||
ports: !reset []
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.insights-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/insights`)
|
||||
- traefik.http.routers.insights-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.insights-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.insights-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.insights-metrics.loadbalancer.server.port=9464
|
||||
|
||||
notifier:
|
||||
ports: !reset []
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.notifier-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/notifier`)
|
||||
- traefik.http.routers.notifier-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.notifier-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.notifier-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.notifier-metrics.loadbalancer.server.port=9464
|
||||
|
||||
audit:
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.routers.audit-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/audit`)
|
||||
- traefik.http.routers.audit-metrics.entrypoints=websecure
|
||||
- traefik.http.routers.audit-metrics.tls.certresolver=le
|
||||
- traefik.http.routers.audit-metrics.middlewares=metrics-auth,metrics-path
|
||||
- traefik.http.services.audit-metrics.loadbalancer.server.port=9464
|
||||
|
||||
# Ships every container's stdout/stderr to the platform's Loki on falken
|
||||
# (k8s/), labelled `container`. Credentials come from .env.observability,
|
||||
|
||||
+14
-8
@@ -39,8 +39,12 @@ OTLP/gRPC (4317) is cluster-internal only. Apps use OTLP/HTTP.
|
||||
ports. `vione.cloud` sits behind Cloudflare's proxy, which only forwards
|
||||
80/443 (plus a few others), so raw ports like `:3000` would never work.
|
||||
- **Resource requests/limits** on every container (falken is shared).
|
||||
- **Not ported:** Prometheus app scrape jobs (`prometheus_extra_scrape_jobs` is
|
||||
empty in the role too; see `prometheus/prometheus.yml`).
|
||||
- **App scrape job added:** `ledgerline` (the role's
|
||||
`prometheus_extra_scrape_jobs` is empty). Prometheus scrapes each czy
|
||||
service at `https://ledgerline.vione.cloud/metrics/<service>`, which czy's
|
||||
Traefik routes to that container's `:9464` behind basic auth. The scrape
|
||||
credentials are the `prometheus-scrape-auth` secret, mounted at
|
||||
`/etc/prometheus/secrets`.
|
||||
|
||||
The CLAUDE.md platform facts still hold:
|
||||
- OTLP metrics and logs are rejected: Alloy returns a 404 on `/v1/metrics` and
|
||||
@@ -53,7 +57,7 @@ The CLAUDE.md platform facts still hold:
|
||||
|
||||
```bash
|
||||
# once: generate credentials (k8s/.credentials, gitignored) and create secrets.
|
||||
# Also writes .env.observability for the czy agent.
|
||||
# Also writes .env.observability and observability/traefik/metrics-users for czy.
|
||||
./k8s/create-secrets.sh
|
||||
|
||||
# deploy, or re-apply after editing any config file
|
||||
@@ -72,10 +76,15 @@ KUBECTL="ssh falken kubectl" ./k8s/create-secrets.sh
|
||||
kubectl kustomize k8s/ | ssh falken kubectl apply -f -
|
||||
```
|
||||
|
||||
**czy agent** (once, then `./deploy.sh` keeps it running):
|
||||
**czy files** (once, then `./deploy.sh` keeps it running). Copy
|
||||
`metrics-users` before the first deploy that mounts it. Otherwise Docker
|
||||
creates a directory at that path, and Traefik's metrics routes fail until you
|
||||
remove it:
|
||||
|
||||
```bash
|
||||
scp .env.observability czy:/opt/apps/LedgerLine/.env.observability
|
||||
ssh czy mkdir -p /opt/apps/LedgerLine/observability/traefik
|
||||
scp observability/traefik/metrics-users czy:/opt/apps/LedgerLine/observability/traefik/metrics-users
|
||||
./deploy.sh
|
||||
```
|
||||
|
||||
@@ -85,7 +94,7 @@ scp .env.observability czy:/opt/apps/LedgerLine/.env.observability
|
||||
kubectl delete -k k8s/ # removes everything, including PVCs (all data)
|
||||
```
|
||||
|
||||
`grafana-admin` and `obs-basic-auth` are deleted with the namespace. Run
|
||||
`grafana-admin`, `obs-basic-auth` and `prometheus-scrape-auth` are deleted with the namespace. Run
|
||||
`create-secrets.sh` again before re-applying; it reuses `k8s/.credentials`.
|
||||
|
||||
## Credentials
|
||||
@@ -110,9 +119,6 @@ OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20$(printf '%s:%s' "$OBS_USER" "
|
||||
|
||||
## Known gaps
|
||||
|
||||
- **App metrics (Phase 5):** Prometheus here can't reach czy's services.
|
||||
Their host ports are stripped by the Traefik override. Decide the path
|
||||
before adding scrape jobs.
|
||||
- **czy host metrics:** node-exporter here watches falken. czy's own
|
||||
saturation (1 vCPU / 1 GB) is invisible until a node-exporter runs there
|
||||
and has a scrape path.
|
||||
|
||||
+20
-3
@@ -9,9 +9,14 @@
|
||||
# obs-basic-auth htpasswd for the Traefik basic-auth middleware
|
||||
# (Prometheus, Alertmanager, Loki, Tempo, Alloy UI, OTLP)
|
||||
# grafana-admin Grafana admin user/password
|
||||
# and writes .env.observability at the repo root (gitignored) for the czy-side
|
||||
# Alloy log agent and the OTLP auth header of instrumented services; copy it to
|
||||
# the server after every regeneration (see k8s/README.md).
|
||||
# prometheus-scrape-auth
|
||||
# username/password Prometheus sends when scraping Ledgerline
|
||||
# and writes two gitignored files for czy; copy both to the server after every
|
||||
# regeneration (see k8s/README.md):
|
||||
# .env.observability for the Alloy log agent and the OTLP auth header of
|
||||
# instrumented services
|
||||
# observability/traefik/metrics-users
|
||||
# htpasswd for Traefik's basic auth on /metrics/<service>
|
||||
#
|
||||
# KUBECTL defaults to `kubectl`; e.g. KUBECTL="ssh falken kubectl" to run it on
|
||||
# the node when the API server isn't reachable from here.
|
||||
@@ -48,6 +53,11 @@ kubectl create secret generic grafana-admin -n "$NS" \
|
||||
--from-literal=admin-password="$GRAFANA_ADMIN_PASSWORD" \
|
||||
--dry-run=client -o yaml | $KUBECTL apply -f -
|
||||
|
||||
kubectl create secret generic prometheus-scrape-auth -n "$NS" \
|
||||
--from-literal=username="$OBS_USER" \
|
||||
--from-literal=password="$OBS_PASSWORD" \
|
||||
--dry-run=client -o yaml | $KUBECTL apply -f -
|
||||
|
||||
# OTLP header for instrumented services: URL-encoded (space -> %20), and
|
||||
# newlines stripped because GNU base64 wraps at 76 chars and this is ~80.
|
||||
OTLP_BASIC=$(printf '%s:%s' "$OBS_USER" "$OBS_PASSWORD" | base64 | tr -d '\n')
|
||||
@@ -64,3 +74,10 @@ OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20$OTLP_BASIC
|
||||
EOF
|
||||
)
|
||||
echo "wrote $(cd .. && pwd)/.env.observability"
|
||||
|
||||
(
|
||||
umask 077
|
||||
mkdir -p ../observability/traefik
|
||||
htpasswd -nbB "$OBS_USER" "$OBS_PASSWORD" > ../observability/traefik/metrics-users
|
||||
)
|
||||
echo "wrote $(cd .. && pwd)/observability/traefik/metrics-users"
|
||||
|
||||
@@ -48,6 +48,9 @@ spec:
|
||||
mountPath: /etc/prometheus/rules
|
||||
- name: data
|
||||
mountPath: /prometheus
|
||||
- name: scrape-auth
|
||||
mountPath: /etc/prometheus/secrets
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
@@ -58,3 +61,10 @@ spec:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data
|
||||
# Basic-auth credentials for the `ledgerline` scrape job, created by
|
||||
# create-secrets.sh. Optional so a missing secret costs that job's
|
||||
# scrapes (up == 0) rather than the whole Prometheus pod.
|
||||
- name: scrape-auth
|
||||
secret:
|
||||
secretName: prometheus-scrape-auth
|
||||
optional: true
|
||||
|
||||
@@ -35,7 +35,38 @@ scrape_configs:
|
||||
static_configs:
|
||||
- targets: ["grafana:3000"]
|
||||
|
||||
# Application scrape jobs go here -- the equivalent of the Ansible role's
|
||||
# `prometheus_extra_scrape_jobs`. Ledgerline runs on czy with its service
|
||||
# ports unpublished, so a reachable path to /metrics has to be decided first
|
||||
# (Phase 5). Re-apply with `kubectl apply -k k8s/` after editing.
|
||||
# Application scrape jobs -- the equivalent of the Ansible role's
|
||||
# `prometheus_extra_scrape_jobs`. Re-apply with `kubectl apply -k k8s/`.
|
||||
#
|
||||
# Ledgerline runs on czy with no service ports published. Each service
|
||||
# serves metrics on its own :9464, which czy's Traefik exposes as
|
||||
# https://ledgerline.vione.cloud/metrics/<service> behind basic auth
|
||||
# (docker-compose.traefik.yml). All seven targets share one address, so the
|
||||
# `service` label is what tells them apart; the relabels derive the path
|
||||
# and a readable instance from it.
|
||||
- job_name: ledgerline
|
||||
scheme: https
|
||||
basic_auth:
|
||||
username_file: /etc/prometheus/secrets/username
|
||||
password_file: /etc/prometheus/secrets/password
|
||||
static_configs:
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: gateway }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: ledger }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: fraud }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: sanctions }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: insights }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: notifier }
|
||||
- targets: ["ledgerline.vione.cloud"]
|
||||
labels: { service: audit }
|
||||
relabel_configs:
|
||||
- source_labels: [service]
|
||||
target_label: __metrics_path__
|
||||
replacement: /metrics/$1
|
||||
- source_labels: [service]
|
||||
target_label: instance
|
||||
|
||||
Reference in New Issue
Block a user