Scrape Ledgerline's metrics from falken through czy's Traefik

czy publishes no service ports, so Traefik exposes each service's :9464
as https://ledgerline.vione.cloud/metrics/<service> behind basic auth,
and the platform's Prometheus gets a `ledgerline` job with one target
per service. The targets share an address; a `service` label tells them
apart, and relabels derive __metrics_path__ and instance from it.

create-secrets.sh now also creates the prometheus-scrape-auth secret
(mounted optional, so a missing secret fails only this job) and writes
the gitignored htpasswd file Traefik reads, which is copied to czy like
.env.observability.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docker-compose.traefik.yml

@@ -5,6 +5,13 @@
 #
 # Only Traefik publishes host ports (80/443). Everything else is reachable
 # only on the compose network.
+#
+# Metrics: every Ledgerline service serves OTel metrics on :9464 (see
+# common/telemetry.py). Traefik exposes each one as
+# https://ledgerline.vione.cloud/metrics/<service>, behind basic auth, so the
+# platform's Prometheus on falken can scrape it (k8s/prometheus/prometheus.yml,
+# job `ledgerline`). The htpasswd file is written by k8s/create-secrets.sh and
+# copied here once, like .env.observability.
 
 services:
   traefik:
@@ -28,6 +35,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - letsencrypt:/letsencrypt
+      - ./observability/traefik/metrics-users:/etc/traefik/metrics-users:ro
 
   gateway:
     ports: !reset []
@@ -37,6 +45,18 @@ services:
       - traefik.http.routers.ledgerline.entrypoints=websecure
       - traefik.http.routers.ledgerline.tls.certresolver=le
       - traefik.http.services.ledgerline.loadbalancer.server.port=8000
+      # gateway now has two Traefik services, so each router names its own.
+      - traefik.http.routers.ledgerline.service=ledgerline
+      - traefik.http.routers.gateway-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/gateway`)
+      - traefik.http.routers.gateway-metrics.entrypoints=websecure
+      - traefik.http.routers.gateway-metrics.tls.certresolver=le
+      - traefik.http.routers.gateway-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.routers.gateway-metrics.service=gateway-metrics
+      - traefik.http.services.gateway-metrics.loadbalancer.server.port=9464
+      # Shared by every <service>-metrics router. Declared here because a
+      # middleware needs an enabled container to carry it.
+      - traefik.http.middlewares.metrics-auth.basicauth.usersfile=/etc/traefik/metrics-users
+      - traefik.http.middlewares.metrics-path.replacepath.path=/metrics
 
   postgres:
     ports: !reset []
@@ -46,18 +66,62 @@ services:
 
   ledger:
     ports: !reset []
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.ledger-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/ledger`)
+      - traefik.http.routers.ledger-metrics.entrypoints=websecure
+      - traefik.http.routers.ledger-metrics.tls.certresolver=le
+      - traefik.http.routers.ledger-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.ledger-metrics.loadbalancer.server.port=9464
 
   fraud:
     ports: !reset []
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.fraud-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/fraud`)
+      - traefik.http.routers.fraud-metrics.entrypoints=websecure
+      - traefik.http.routers.fraud-metrics.tls.certresolver=le
+      - traefik.http.routers.fraud-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.fraud-metrics.loadbalancer.server.port=9464
 
   sanctions:
     ports: !reset []
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.sanctions-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/sanctions`)
+      - traefik.http.routers.sanctions-metrics.entrypoints=websecure
+      - traefik.http.routers.sanctions-metrics.tls.certresolver=le
+      - traefik.http.routers.sanctions-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.sanctions-metrics.loadbalancer.server.port=9464
 
   insights:
     ports: !reset []
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.insights-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/insights`)
+      - traefik.http.routers.insights-metrics.entrypoints=websecure
+      - traefik.http.routers.insights-metrics.tls.certresolver=le
+      - traefik.http.routers.insights-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.insights-metrics.loadbalancer.server.port=9464
 
   notifier:
     ports: !reset []
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.notifier-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/notifier`)
+      - traefik.http.routers.notifier-metrics.entrypoints=websecure
+      - traefik.http.routers.notifier-metrics.tls.certresolver=le
+      - traefik.http.routers.notifier-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.notifier-metrics.loadbalancer.server.port=9464
+
+  audit:
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.audit-metrics.rule=Host(`ledgerline.vione.cloud`) && Path(`/metrics/audit`)
+      - traefik.http.routers.audit-metrics.entrypoints=websecure
+      - traefik.http.routers.audit-metrics.tls.certresolver=le
+      - traefik.http.routers.audit-metrics.middlewares=metrics-auth,metrics-path
+      - traefik.http.services.audit-metrics.loadbalancer.server.port=9464
 
   # Ships every container's stdout/stderr to the platform's Loki on falken
   # (k8s/), labelled `container`. Credentials come from .env.observability,