diff --git a/.gitignore b/.gitignore index 6c057f9..c335733 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ .pytest_cache/ .env .env.observability +observability/traefik/metrics-users k8s/.credentials *.egg-info/ .DS_Store diff --git a/CLAUDE.md b/CLAUDE.md index fac8e20..54f4b5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,8 +67,9 @@ at one point in time and the user may have since extended them. 2. **App metrics reach Prometheus by direct scrape, not through Alloy.** Alloy has no `prometheus.scrape` component; `k8s/prometheus/prometheus.yml` scrapes the stack's own components, and app jobs get added there (the role's - `prometheus_extra_scrape_jobs`). czy's service ports are unpublished, so no app - scrape path exists yet — a Phase 5 decision. Note that + `prometheus_extra_scrape_jobs`). czy's service ports are unpublished, so the + `ledgerline` job scrapes each service's `:9464` through czy's Traefik at + `https://ledgerline.vione.cloud/metrics/` (basic auth). Note that the diagram draws this differently (app metrics routed via Alloy) — the diagram and the code have drifted, and the user knows. Don't "correct" one to the other unasked. diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml index d288a21..8bcde55 100644 --- a/docker-compose.traefik.yml +++ b/docker-compose.traefik.yml @@ -5,6 +5,13 @@ # # Only Traefik publishes host ports (80/443). Everything else is reachable # only on the compose network. +# +# Metrics: every Ledgerline service serves OTel metrics on :9464 (see +# common/telemetry.py). Traefik exposes each one as +# https://ledgerline.vione.cloud/metrics/, behind basic auth, so the +# platform's Prometheus on falken can scrape it (k8s/prometheus/prometheus.yml, +# job `ledgerline`). The htpasswd file is written by k8s/create-secrets.sh and +# copied here once, like .env.observability. services: traefik: @@ -28,6 +35,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - letsencrypt:/letsencrypt + - ./observability/traefik/metrics-users:/etc/traefik/metrics-users:ro gateway: ports: !reset [] @@ -37,6 +45,18 @@ services: - traefik.http.routers.ledgerline.entrypoints=websecure - traefik.http.routers.ledgerline.tls.certresolver=le - traefik.http.services.ledgerline.loadbalancer.server.port=8000 + # gateway now has two Traefik services, so each router names its own. + - traefik.http.routers.ledgerline.service=ledgerline + - traefik.http.routers.gateway-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/gateway`) + - traefik.http.routers.gateway-metrics.entrypoints=websecure + - traefik.http.routers.gateway-metrics.tls.certresolver=le + - traefik.http.routers.gateway-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.routers.gateway-metrics.service=gateway-metrics + - traefik.http.services.gateway-metrics.loadbalancer.server.port=9464 + # Shared by every -metrics router. Declared here because a + # middleware needs an enabled container to carry it. + - traefik.http.middlewares.metrics-auth.basicauth.usersfile=/etc/traefik/metrics-users + - traefik.http.middlewares.metrics-path.replacepath.path=/metrics postgres: ports: !reset [] @@ -46,18 +66,62 @@ services: ledger: ports: !reset [] + labels: + - traefik.enable=true + - traefik.http.routers.ledger-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/ledger`) + - traefik.http.routers.ledger-metrics.entrypoints=websecure + - traefik.http.routers.ledger-metrics.tls.certresolver=le + - traefik.http.routers.ledger-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.ledger-metrics.loadbalancer.server.port=9464 fraud: ports: !reset [] + labels: + - traefik.enable=true + - traefik.http.routers.fraud-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/fraud`) + - traefik.http.routers.fraud-metrics.entrypoints=websecure + - traefik.http.routers.fraud-metrics.tls.certresolver=le + - traefik.http.routers.fraud-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.fraud-metrics.loadbalancer.server.port=9464 sanctions: ports: !reset [] + labels: + - traefik.enable=true + - traefik.http.routers.sanctions-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/sanctions`) + - traefik.http.routers.sanctions-metrics.entrypoints=websecure + - traefik.http.routers.sanctions-metrics.tls.certresolver=le + - traefik.http.routers.sanctions-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.sanctions-metrics.loadbalancer.server.port=9464 insights: ports: !reset [] + labels: + - traefik.enable=true + - traefik.http.routers.insights-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/insights`) + - traefik.http.routers.insights-metrics.entrypoints=websecure + - traefik.http.routers.insights-metrics.tls.certresolver=le + - traefik.http.routers.insights-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.insights-metrics.loadbalancer.server.port=9464 notifier: ports: !reset [] + labels: + - traefik.enable=true + - traefik.http.routers.notifier-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/notifier`) + - traefik.http.routers.notifier-metrics.entrypoints=websecure + - traefik.http.routers.notifier-metrics.tls.certresolver=le + - traefik.http.routers.notifier-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.notifier-metrics.loadbalancer.server.port=9464 + + audit: + labels: + - traefik.enable=true + - traefik.http.routers.audit-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/audit`) + - traefik.http.routers.audit-metrics.entrypoints=websecure + - traefik.http.routers.audit-metrics.tls.certresolver=le + - traefik.http.routers.audit-metrics.middlewares=metrics-auth,metrics-path + - traefik.http.services.audit-metrics.loadbalancer.server.port=9464 # Ships every container's stdout/stderr to the platform's Loki on falken # (k8s/), labelled `container`. Credentials come from .env.observability, diff --git a/k8s/README.md b/k8s/README.md index 5834549..c2d66ea 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -39,8 +39,12 @@ OTLP/gRPC (4317) is cluster-internal only. Apps use OTLP/HTTP. ports. `vione.cloud` sits behind Cloudflare's proxy, which only forwards 80/443 (plus a few others), so raw ports like `:3000` would never work. - **Resource requests/limits** on every container (falken is shared). -- **Not ported:** Prometheus app scrape jobs (`prometheus_extra_scrape_jobs` is - empty in the role too; see `prometheus/prometheus.yml`). +- **App scrape job added:** `ledgerline` (the role's + `prometheus_extra_scrape_jobs` is empty). Prometheus scrapes each czy + service at `https://ledgerline.vione.cloud/metrics/`, which czy's + Traefik routes to that container's `:9464` behind basic auth. The scrape + credentials are the `prometheus-scrape-auth` secret, mounted at + `/etc/prometheus/secrets`. The CLAUDE.md platform facts still hold: - OTLP metrics and logs are rejected: Alloy returns a 404 on `/v1/metrics` and @@ -53,7 +57,7 @@ The CLAUDE.md platform facts still hold: ```bash # once: generate credentials (k8s/.credentials, gitignored) and create secrets. -# Also writes .env.observability for the czy agent. +# Also writes .env.observability and observability/traefik/metrics-users for czy. ./k8s/create-secrets.sh # deploy, or re-apply after editing any config file @@ -72,10 +76,15 @@ KUBECTL="ssh falken kubectl" ./k8s/create-secrets.sh kubectl kustomize k8s/ | ssh falken kubectl apply -f - ``` -**czy agent** (once, then `./deploy.sh` keeps it running): +**czy files** (once, then `./deploy.sh` keeps it running). Copy +`metrics-users` before the first deploy that mounts it. Otherwise Docker +creates a directory at that path, and Traefik's metrics routes fail until you +remove it: ```bash scp .env.observability czy:/opt/apps/LedgerLine/.env.observability +ssh czy mkdir -p /opt/apps/LedgerLine/observability/traefik +scp observability/traefik/metrics-users czy:/opt/apps/LedgerLine/observability/traefik/metrics-users ./deploy.sh ``` @@ -85,7 +94,7 @@ scp .env.observability czy:/opt/apps/LedgerLine/.env.observability kubectl delete -k k8s/ # removes everything, including PVCs (all data) ``` -`grafana-admin` and `obs-basic-auth` are deleted with the namespace. Run +`grafana-admin`, `obs-basic-auth` and `prometheus-scrape-auth` are deleted with the namespace. Run `create-secrets.sh` again before re-applying; it reuses `k8s/.credentials`. ## Credentials @@ -110,9 +119,6 @@ OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20$(printf '%s:%s' "$OBS_USER" " ## Known gaps -- **App metrics (Phase 5):** Prometheus here can't reach czy's services. - Their host ports are stripped by the Traefik override. Decide the path - before adding scrape jobs. - **czy host metrics:** node-exporter here watches falken. czy's own saturation (1 vCPU / 1 GB) is invisible until a node-exporter runs there and has a scrape path. diff --git a/k8s/create-secrets.sh b/k8s/create-secrets.sh index 80dc393..557809e 100755 --- a/k8s/create-secrets.sh +++ b/k8s/create-secrets.sh @@ -9,9 +9,14 @@ # obs-basic-auth htpasswd for the Traefik basic-auth middleware # (Prometheus, Alertmanager, Loki, Tempo, Alloy UI, OTLP) # grafana-admin Grafana admin user/password -# and writes .env.observability at the repo root (gitignored) for the czy-side -# Alloy log agent and the OTLP auth header of instrumented services; copy it to -# the server after every regeneration (see k8s/README.md). +# prometheus-scrape-auth +# username/password Prometheus sends when scraping Ledgerline +# and writes two gitignored files for czy; copy both to the server after every +# regeneration (see k8s/README.md): +# .env.observability for the Alloy log agent and the OTLP auth header of +# instrumented services +# observability/traefik/metrics-users +# htpasswd for Traefik's basic auth on /metrics/ # # KUBECTL defaults to `kubectl`; e.g. KUBECTL="ssh falken kubectl" to run it on # the node when the API server isn't reachable from here. @@ -48,6 +53,11 @@ kubectl create secret generic grafana-admin -n "$NS" \ --from-literal=admin-password="$GRAFANA_ADMIN_PASSWORD" \ --dry-run=client -o yaml | $KUBECTL apply -f - +kubectl create secret generic prometheus-scrape-auth -n "$NS" \ + --from-literal=username="$OBS_USER" \ + --from-literal=password="$OBS_PASSWORD" \ + --dry-run=client -o yaml | $KUBECTL apply -f - + # OTLP header for instrumented services: URL-encoded (space -> %20), and # newlines stripped because GNU base64 wraps at 76 chars and this is ~80. OTLP_BASIC=$(printf '%s:%s' "$OBS_USER" "$OBS_PASSWORD" | base64 | tr -d '\n') @@ -64,3 +74,10 @@ OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20$OTLP_BASIC EOF ) echo "wrote $(cd .. && pwd)/.env.observability" + +( + umask 077 + mkdir -p ../observability/traefik + htpasswd -nbB "$OBS_USER" "$OBS_PASSWORD" > ../observability/traefik/metrics-users +) +echo "wrote $(cd .. && pwd)/observability/traefik/metrics-users" diff --git a/k8s/prometheus/deployment.yaml b/k8s/prometheus/deployment.yaml index f598141..e01bfdb 100644 --- a/k8s/prometheus/deployment.yaml +++ b/k8s/prometheus/deployment.yaml @@ -48,6 +48,9 @@ spec: mountPath: /etc/prometheus/rules - name: data mountPath: /prometheus + - name: scrape-auth + mountPath: /etc/prometheus/secrets + readOnly: true volumes: - name: config configMap: @@ -58,3 +61,10 @@ spec: - name: data persistentVolumeClaim: claimName: prometheus-data + # Basic-auth credentials for the `ledgerline` scrape job, created by + # create-secrets.sh. Optional so a missing secret costs that job's + # scrapes (up == 0) rather than the whole Prometheus pod. + - name: scrape-auth + secret: + secretName: prometheus-scrape-auth + optional: true diff --git a/k8s/prometheus/prometheus.yml b/k8s/prometheus/prometheus.yml index 6d6e1a6..592a45a 100644 --- a/k8s/prometheus/prometheus.yml +++ b/k8s/prometheus/prometheus.yml @@ -35,7 +35,38 @@ scrape_configs: static_configs: - targets: ["grafana:3000"] - # Application scrape jobs go here -- the equivalent of the Ansible role's - # `prometheus_extra_scrape_jobs`. Ledgerline runs on czy with its service - # ports unpublished, so a reachable path to /metrics has to be decided first - # (Phase 5). Re-apply with `kubectl apply -k k8s/` after editing. + # Application scrape jobs -- the equivalent of the Ansible role's + # `prometheus_extra_scrape_jobs`. Re-apply with `kubectl apply -k k8s/`. + # + # Ledgerline runs on czy with no service ports published. Each service + # serves metrics on its own :9464, which czy's Traefik exposes as + # https://ledgerline.vione.cloud/metrics/ behind basic auth + # (docker-compose.traefik.yml). All seven targets share one address, so the + # `service` label is what tells them apart; the relabels derive the path + # and a readable instance from it. + - job_name: ledgerline + scheme: https + basic_auth: + username_file: /etc/prometheus/secrets/username + password_file: /etc/prometheus/secrets/password + static_configs: + - targets: ["ledgerline.vione.cloud"] + labels: { service: gateway } + - targets: ["ledgerline.vione.cloud"] + labels: { service: ledger } + - targets: ["ledgerline.vione.cloud"] + labels: { service: fraud } + - targets: ["ledgerline.vione.cloud"] + labels: { service: sanctions } + - targets: ["ledgerline.vione.cloud"] + labels: { service: insights } + - targets: ["ledgerline.vione.cloud"] + labels: { service: notifier } + - targets: ["ledgerline.vione.cloud"] + labels: { service: audit } + relabel_configs: + - source_labels: [service] + target_label: __metrics_path__ + replacement: /metrics/$1 + - source_labels: [service] + target_label: instance