From 7719ef5fcf149cd32981426bd829d608af6ed0ba Mon Sep 17 00:00:00 2001 From: Movses Aslanyan Date: Mon, 12 Oct 2026 13:24:38 +0330 Subject: [PATCH] Port the observability platform to k3s and ship czy logs to it k8s/ is now a faithful port of the mentor's Ansible Grafana stack (Prometheus, Alertmanager, Loki, Tempo, Alloy, Grafana, node-exporter) for the ledgerline-observability namespace on falken: same versions and configs, one *.vione.cloud ingress per tool behind Traefik basic auth, secrets generated by create-secrets.sh into gitignored files. Two fixes over the Ansible role, both verified live: Tempo's metrics-generator needs traces_storage or it runs no processors, and Prometheus needs exemplar-storage to keep exemplars. Ledgerline stays on czy; an Alloy agent in the Traefik override ships its container logs to Loki. Roadmap and CLAUDE.md now point at k8s/. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + CLAUDE.md | 90 +++++++++++++------ docker-compose.traefik.yml | 21 +++++ docs/roadmap/intro.md | 66 ++++++++++---- docs/roadmap/phase-11.md | 4 +- docs/roadmap/phase-2.md | 84 +++++++++++------- docs/roadmap/phase-4.md | 11 ++- docs/roadmap/phase-5.md | 20 +++-- docs/roadmap/phase-6.md | 29 +++--- docs/roadmap/phase-8.md | 6 +- docs/roadmap/summary.md | 11 +-- k8s/README.md | 117 +++++++++++++++++++++++++ k8s/alertmanager/alertmanager.yml | 17 ++++ k8s/alertmanager/deployment.yaml | 46 ++++++++++ k8s/alertmanager/ingress.yaml | 24 +++++ k8s/alertmanager/kustomization.yaml | 13 +++ k8s/alertmanager/pvc.yaml | 11 +++ k8s/alertmanager/service.yaml | 12 +++ k8s/alloy/config.alloy | 35 ++++++++ k8s/alloy/deployment.yaml | 47 ++++++++++ k8s/alloy/ingress.yaml | 42 +++++++++ k8s/alloy/kustomization.yaml | 12 +++ k8s/alloy/service.yaml | 18 ++++ k8s/auth/kustomization.yaml | 5 ++ k8s/auth/middleware.yaml | 14 +++ k8s/create-secrets.sh | 60 +++++++++++++ k8s/grafana/datasources.yml | 53 +++++++++++ k8s/grafana/deployment.yaml | 60 +++++++++++++ k8s/grafana/ingress.yaml | 24 +++++ k8s/grafana/kustomization.yaml | 15 ++++ k8s/grafana/pvc.yaml | 11 +++ k8s/grafana/service.yaml | 12 +++ k8s/kustomization.yaml | 26 ++++++ k8s/loki/deployment.yaml | 46 ++++++++++ k8s/loki/ingress.yaml | 26 ++++++ k8s/loki/kustomization.yaml | 13 +++ k8s/loki/loki.yml | 33 +++++++ k8s/loki/pvc.yaml | 11 +++ k8s/loki/service.yaml | 12 +++ k8s/namespace.yaml | 4 + k8s/node-exporter/daemonset.yaml | 44 ++++++++++ k8s/node-exporter/kustomization.yaml | 6 ++ k8s/node-exporter/service.yaml | 12 +++ k8s/prometheus/deployment.yaml | 60 +++++++++++++ k8s/prometheus/ingress.yaml | 24 +++++ k8s/prometheus/kustomization.yaml | 16 ++++ k8s/prometheus/prometheus.yml | 41 +++++++++ k8s/prometheus/pvc.yaml | 11 +++ k8s/prometheus/rules/host.rules.yml | 20 +++++ k8s/prometheus/service.yaml | 12 +++ k8s/tempo/deployment.yaml | 48 ++++++++++ k8s/tempo/ingress.yaml | 26 ++++++ k8s/tempo/kustomization.yaml | 13 +++ k8s/tempo/pvc.yaml | 11 +++ k8s/tempo/service.yaml | 18 ++++ k8s/tempo/tempo.yml | 47 ++++++++++ observability/alloy-agent/config.alloy | 45 ++++++++++ 57 files changed, 1507 insertions(+), 110 deletions(-) create mode 100644 k8s/README.md create mode 100644 k8s/alertmanager/alertmanager.yml create mode 100644 k8s/alertmanager/deployment.yaml create mode 100644 k8s/alertmanager/ingress.yaml create mode 100644 k8s/alertmanager/kustomization.yaml create mode 100644 k8s/alertmanager/pvc.yaml create mode 100644 k8s/alertmanager/service.yaml create mode 100644 k8s/alloy/config.alloy create mode 100644 k8s/alloy/deployment.yaml create mode 100644 k8s/alloy/ingress.yaml create mode 100644 k8s/alloy/kustomization.yaml create mode 100644 k8s/alloy/service.yaml create mode 100644 k8s/auth/kustomization.yaml create mode 100644 k8s/auth/middleware.yaml create mode 100755 k8s/create-secrets.sh create mode 100644 k8s/grafana/datasources.yml create mode 100644 k8s/grafana/deployment.yaml create mode 100644 k8s/grafana/ingress.yaml create mode 100644 k8s/grafana/kustomization.yaml create mode 100644 k8s/grafana/pvc.yaml create mode 100644 k8s/grafana/service.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/loki/deployment.yaml create mode 100644 k8s/loki/ingress.yaml create mode 100644 k8s/loki/kustomization.yaml create mode 100644 k8s/loki/loki.yml create mode 100644 k8s/loki/pvc.yaml create mode 100644 k8s/loki/service.yaml create mode 100644 k8s/namespace.yaml create mode 100644 k8s/node-exporter/daemonset.yaml create mode 100644 k8s/node-exporter/kustomization.yaml create mode 100644 k8s/node-exporter/service.yaml create mode 100644 k8s/prometheus/deployment.yaml create mode 100644 k8s/prometheus/ingress.yaml create mode 100644 k8s/prometheus/kustomization.yaml create mode 100644 k8s/prometheus/prometheus.yml create mode 100644 k8s/prometheus/pvc.yaml create mode 100644 k8s/prometheus/rules/host.rules.yml create mode 100644 k8s/prometheus/service.yaml create mode 100644 k8s/tempo/deployment.yaml create mode 100644 k8s/tempo/ingress.yaml create mode 100644 k8s/tempo/kustomization.yaml create mode 100644 k8s/tempo/pvc.yaml create mode 100644 k8s/tempo/service.yaml create mode 100644 k8s/tempo/tempo.yml create mode 100644 observability/alloy-agent/config.alloy diff --git a/.gitignore b/.gitignore index 3f278f1..6c057f9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ __pycache__/ *.pyo .pytest_cache/ .env +.env.observability +k8s/.credentials *.egg-info/ .DS_Store venv/ diff --git a/CLAUDE.md b/CLAUDE.md index d03e78a..743ed8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,16 +25,30 @@ below. Phase 0 is built test-first from two specs: `docs/phase-0a-test-suite-spec.md` (the tests) and `docs/phase-0-base-project-spec.md` (the system). -### The observability backend is external and pre-built +### The observability backend is pre-built, ported to k3s -The Grafana stack is **not** built in this repo. It is deployed by a separate -Ansible project, `observability-ansible`, which the user received from his mentor. -It deploys two stacks on a target host via Docker Compose: +The platform came from the mentor as an Ansible project, `observability-ansible` +(kept in `ansible/` as the reference). It deploys two stacks on one host via +Docker Compose: - **Grafana stack:** Prometheus + Alertmanager + Loki + Tempo + Grafana, with **Grafana Alloy** as the collector and node-exporter for host metrics. - **ELK stack:** Filebeat → Logstash → Elasticsearch → Kibana/Grafana. This is the - hands-on subject of Phase 10, not part of the main pipeline. + hands-on subject of Phase 10, not part of the main pipeline. **Not ported.** + +**The Grafana stack actually runs on the user's k3s cluster (`falken`), not via +Ansible.** `k8s/` is a faithful port of the role: same versions, configs and +pipeline, namespace `ledgerline-observability`, deployed with +`kubectl apply -k k8s/` (`k` is the user's alias with the right kubeconfig; if +the API server is unreachable, `kubectl kustomize k8s/ | ssh falken kubectl +apply -f -`). Endpoints are `https://{grafana,prometheus,alertmanager,loki,tempo,alloy,otlp}.vione.cloud`; +all but Grafana sit behind basic auth with credentials in `k8s/.credentials`. +Ledgerline stays on `czy` (Compose); its container logs reach Loki via an Alloy +agent there (`observability/alloy-agent/config.alloy`, in +`docker-compose.traefik.yml`) — the Docker-socket half of the original Alloy +config. `k8s/README.md` lists every deviation from the Ansible role. Where a +roadmap file still says "Ansible role" or "re-run the playbook", read it as +"`k8s/`" and "`kubectl apply -k k8s/`". Five facts about that platform that change what correct code looks like here. Verify them against the live system rather than assuming — they were read from the configs @@ -47,8 +61,10 @@ at one point in time and the user may have since extended them. separate `/metrics` Prometheus-exposition input. Never tell the user their OTLP metrics "should be arriving"; check Prometheus or the Alloy UI first. 2. **App metrics reach Prometheus by direct scrape, not through Alloy.** Alloy has - no `prometheus.scrape` component; `prometheus.yml.j2` scrapes the stack's own - components plus whatever is listed in `prometheus_extra_scrape_jobs`. Note that + no `prometheus.scrape` component; `k8s/prometheus/prometheus.yml` scrapes the + stack's own components, and app jobs get added there (the role's + `prometheus_extra_scrape_jobs`). czy's service ports are unpublished, so no app + scrape path exists yet — a Phase 5 decision. Note that the diagram draws this differently (app metrics routed via Alloy) — the diagram and the code have drifted, and the user knows. Don't "correct" one to the other unasked. @@ -57,15 +73,20 @@ at one point in time and the user may have since extended them. Prometheus with `send_exemplars: true`) — **not** from collector connectors. Never add `spanmetrics` / `servicegraph` to Alloy; it duplicates every series. The metric names differ from the ones in most blog posts — look them up in - Prometheus before writing PromQL. + Prometheus before writing PromQL. (As shipped, the Ansible role produced none of + this: Tempo lacked `traces_storage` and Prometheus lacked + `--enable-feature=exemplar-storage`. Both are fixed in `k8s/`; verified live.) 4. **Log→trace correlation runs on a case-sensitive regex:** `trace[_-]?id["=: ]+([a-fA-F0-9]+)`. Logs must carry **lowercase `trace_id`**. OpenTelemetry's logging auto-instrumentation emits `otelTraceID`, which does not match. Exemplar labels must also be `trace_id`. The diagram names this outright: - "Correlation key: trace_id / exemplars." + "Correlation key: trace_id / exemplars." **Known, unresolved:** Tempo's + metrics-generator labels its exemplars `traceID` (verified live), which the + provisioned `exemplarTraceIdDestinations: name: trace_id` does not match — so + span-metric exemplars don't link to Tempo yet. The user decides the fix. 5. **There is no dashboard provisioning in the platform** — datasources only. - Dashboards-as-code is the user's deliverable and belongs in the Ansible role, so - that `ansible-playbook` rebuilds them. + Dashboards-as-code is the user's deliverable and belongs in `k8s/grafana/`, so + that `kubectl apply -k k8s/` rebuilds them. Ledgerline talks to this platform over the network. Endpoints come from env (`OTEL_EXPORTER_OTLP_ENDPOINT` etc.), never hardcoded. @@ -294,42 +315,53 @@ git diff --stat tests/ # did anything touch the contract? should The pathology suite (`make test-pathology`, `make test-slow`) is slow and is an answer key — see spoiler rule 3 before running it. -**Traces (Tempo, :3200)** — search by service and duration rather than trusting +**The platform lives on falken** (see the backend section above). Load the +credentials first; every endpoint except Grafana needs basic auth: +```bash +source k8s/.credentials; A="$OBS_USER:$OBS_PASSWORD" +k get pods -n ledgerline-observability # what's actually running +``` + +**Traces (Tempo)** — search by service and duration rather than trusting that a span exists: ```bash -curl -s 'http://localhost:3200/api/search?tags=service.name%3Dledger&limit=5' -curl -s "http://localhost:3200/api/traces/" | jq '.batches | length' +curl -s -u "$A" 'https://tempo.vione.cloud/api/search?tags=service.name%3Dledger&limit=5' +curl -s -u "$A" "https://tempo.vione.cloud/api/traces/" | jq '.batches | length' ``` -**Metrics (Prometheus, :9090)** — and this is the cardinality check worth +**Metrics (Prometheus)** — and this is the cardinality check worth running often: ```bash -curl -s 'http://localhost:9090/api/v1/query?query=count({__name__=~"ledgerline.*"})' -curl -s 'http://localhost:9090/api/v1/status/tsdb' | jq '.data.seriesCountByMetricName[:10]' +curl -s -u "$A" 'https://prometheus.vione.cloud/api/v1/query?query=count({__name__=~"ledgerline.*"})' +curl -s -u "$A" 'https://prometheus.vione.cloud/api/v1/status/tsdb' | jq '.data.seriesCountByMetricName[:10]' ``` -**Logs (Loki, :3100)** — confirm `trace_id` is actually in the payload, not just -in the code that was supposed to put it there: +**Logs (Loki)** — confirm `trace_id` is actually in the payload, not just +in the code that was supposed to put it there. The czy agent labels streams with +the full container name (`container` and `service_name` are both e.g. +`ledgerline-gateway-1`): ```bash -curl -s -G 'http://localhost:3100/loki/api/v1/query_range' \ - --data-urlencode 'query={service_name="gateway"} | json | trace_id != ""' \ +curl -s -u "$A" -G 'https://loki.vione.cloud/loki/api/v1/query_range' \ + --data-urlencode 'query={container="ledgerline-gateway-1"} | json | trace_id != ""' \ --data-urlencode 'limit=5' ``` -**Alloy (:12345)** — the component graph is the fastest way to see what the pipeline -actually does, including which signal outputs are wired. Check it before theorising -about missing data. +**Alloy (https://alloy.vione.cloud)** — the component graph is the fastest way to +see what the pipeline actually does, including which signal outputs are wired. +Check it before theorising about missing data. The czy log agent has no UI +exposed; `ssh czy docker logs ledgerline-alloy-agent-1`. **Metric names** — never assume; the Tempo-generated names differ from blog posts: ```bash -curl -s 'http://:9090/api/v1/label/__name__/values' | jq '.data[] | select(test("trace|span|service_graph"))' +curl -s -u "$A" 'https://prometheus.vione.cloud/api/v1/label/__name__/values' | jq '.data[] | select(test("trace|span|service_graph"))' ``` -**Grafana (:3000)** — verify provisioning actually loaded rather than reading -the YAML and assuming: +**Grafana (https://grafana.vione.cloud)** — verify provisioning actually loaded +rather than reading the YAML and assuming: ```bash -curl -s -u admin:admin 'http://localhost:3000/api/datasources' | jq '.[].name' -curl -s -u admin:admin 'http://localhost:3000/api/search?type=dash-db' | jq '.[].title' +G="$GRAFANA_ADMIN_USER:$GRAFANA_ADMIN_PASSWORD" +curl -s -u "$G" 'https://grafana.vione.cloud/api/datasources' | jq '.[].name' +curl -s -u "$G" 'https://grafana.vione.cloud/api/search?type=dash-db' | jq '.[].title' ``` Rules of thumb: diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml index 5b01949..d288a21 100644 --- a/docker-compose.traefik.yml +++ b/docker-compose.traefik.yml @@ -59,5 +59,26 @@ services: notifier: ports: !reset [] + # Ships every container's stdout/stderr to the platform's Loki on falken + # (k8s/), labelled `container`. Credentials come from .env.observability, + # generated by k8s/create-secrets.sh and copied to the server once. + # Publishes no ports; memory-capped because czy has 1 GB of RAM. Steady + # state is ~70 MiB, but the initial backfill of every container's log + # history peaked at ~150 MiB in testing, hence the headroom. + alloy-agent: + image: grafana/alloy:v1.10.2 + restart: unless-stopped + command: + - run + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + env_file: .env.observability + mem_limit: 256m + volumes: + - ./observability/alloy-agent/config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - alloy-agent-data:/var/lib/alloy/data + volumes: letsencrypt: + alloy-agent-data: diff --git a/docs/roadmap/intro.md b/docs/roadmap/intro.md index b945241..8eee340 100644 --- a/docs/roadmap/intro.md +++ b/docs/roadmap/intro.md @@ -153,6 +153,20 @@ Deployment: `ansible-playbook playbooks/grafana-stack.yml` and (Grafana 12.1.1, Prometheus v3.5.0, Loki 3.5.3, Tempo 2.8.2, Alloy v1.10.2, Elastic 9.1.3). Everything lands under `/opt/observability/`. +> **How it actually runs here (since 2026-10-12):** the Grafana stack is ported +> to the k3s cluster `falken` as **`k8s/`** (same versions, configs and pipeline) +> and deployed with `kubectl apply -k k8s/`; the Ansible project stays in +> `ansible/` as the reference. Everything below describes the Ansible design and +> still holds for the port. Where it names a template, the k8s equivalent is: +> `config.alloy.j2` → `k8s/alloy/config.alloy` (plus the Docker-log half in +> `observability/alloy-agent/config.alloy` on czy), `prometheus.yml.j2` / +> `prometheus_extra_scrape_jobs` → `k8s/prometheus/prometheus.yml`, +> `tempo.yml.j2` → `k8s/tempo/tempo.yml`, `datasources.yml.j2` → +> `k8s/grafana/datasources.yml`, `alertmanager.yml.j2` → +> `k8s/alertmanager/alertmanager.yml`, "re-run the playbook" → +> `kubectl apply -k k8s/`. Deviations are listed in `k8s/README.md`. The ELK +> stack is not ported. + ### The architecture Per `diagrams/02-grafana-lgtm-operational.svg`, the platform is five columns: @@ -266,6 +280,14 @@ names. The metric names differ from the OneUptime article's; go find the actual names in Prometheus before writing any PromQL. (`local-blocks` also gives you TraceQL metrics queries in Tempo — a bonus worth exploring in Phase 9.) +*Verified live on 2026-10-12: as shipped, none of this worked.* `local-blocks` +needs `metrics_generator.traces_storage`. Without it Tempo 2.8 logs "local blocks +processor requires traces wal" and starts **no** processors, so there are no span +metrics and no service graph. Separately, Prometheus throws away remote-written +exemplars unless it runs with `--enable-feature=exemplar-storage`. Both are fixed +in `k8s/`. It's worth telling Babak, and it's a good example of "config present +≠ data arriving". + **3. Log→trace correlation depends on an exact, case-sensitive regex.** The Loki datasource is provisioned with: @@ -285,10 +307,17 @@ datasource declares `exemplarTraceIdDestinations: name: trace_id`. The diagram s this outright in its legend: **"Correlation key: trace_id / exemplars."** Everything in the Explore & Correlate column hangs off that one field name. +*Verified live:* Tempo's metrics-generator labels the exemplars it writes +**`traceID`**, not `trace_id`. So the provisioned exemplar destination does not +match the span-metric exemplars the platform itself produces, and clicking one +won't open the trace. This is left unresolved on purpose (Phase 5 material). +Work out the options and pick one. + **4. There is no dashboard provisioning at all.** The Grafana provisioning mount contains only `datasources/`. Every dashboard is yours to add — which means Babak's "create dashboards as code, destroy and recreate in seconds" requirement is entirely -your deliverable, and the natural home for it is *inside this Ansible role*. +your deliverable, and the natural home for it is *next to the platform's own +provisioning*: originally the Ansible role, now `k8s/grafana/`. ### What this changes, phase by phase @@ -296,38 +325,41 @@ your deliverable, and the natural home for it is *inside this Ansible role*. |---|---| | 0 / 0a | **None.** App and test suite are unaffected. | | 1 | **New.** Deploy Ledgerline itself to a server with Docker Compose behind Traefik. Plumbing only — no telemetry, no app changes. | -| 2 | Replaced. Deploy the real stack with Ansible instead of running a single container. | +| 2 | Replaced. Deploy the real stack (ported to k3s as `k8s/`) instead of running a single container. | | 3 | Endpoint changes to Alloy; tracing work itself is identical. | | 4 | Rewritten. OTLP logs go nowhere; stdout scraping is the path. Lowercase `trace_id` is mandatory. | | 5 | Rewritten. Tempo generates span metrics; app metrics need a scrape endpoint or an Alloy extension. | -| 6 | **Upgraded.** Dashboards-as-code becomes an Ansible deliverable — a better answer than a Makefile target. | +| 6 | **Upgraded.** Dashboards-as-code ships with the platform (`k8s/grafana/`, rebuilt by `kubectl apply -k k8s/`) — a better answer than a Makefile target. | | 7 | None. | | 8 | Choice added: Prometheus rules + Alertmanager (the platform's grain) vs Grafana unified alerting. | | 9 | None, plus TraceQL metrics from `local-blocks`. | | 10 | **Massively upgraded.** You now have a real Filebeat → Logstash → Elasticsearch pipeline to run, not a thought experiment. | -| 11 | Demo now includes `ansible-playbook` rebuilding the platform. | +| 11 | Demo now includes `kubectl apply -k k8s/` rebuilding the platform. | -### Where to run it +### Where it runs The README's own sizing is 2 vCPU / 4 GB for the Grafana stack and 4 vCPU / 8 GB for -ELK. Your single-node k8s box is 4 CPU / 8 GB total — running ELK there alongside -everything else will hurt. Use a separate VM (a cheap cloud instance or a local -Multipass/UTM VM) as the Ansible target, and keep Ledgerline on your laptop pointing -at it. That also makes the deployment *real* rather than a localhost shortcut, which -is the point of it being Ansible. +ELK. Ledgerline runs on `czy` (1 vCPU / 1 GB, Phase 1), which has no room for +either. The Grafana stack therefore runs on the single-node k3s cluster `falken` +(4 CPU / 8 GB, shared with other workloads) in namespace +`ledgerline-observability`, with resource requests and limits set. ELK alongside +it would hurt; decide where it goes when Phase 10 comes. -If you do colocate, note that Alloy's `loki.source.docker` reads the host's Docker -socket and will scrape **every** container's stdout, including Ledgerline's, -automatically — convenient, and worth knowing before you wonder where those logs -came from. +In the Ansible design, Alloy's `loki.source.docker` reads the *platform host's* +Docker socket. That only works when the apps run on that host. Here they don't, +so that half of the config runs as a small Alloy agent on `czy` +(`observability/alloy-agent/config.alloy`). It picks up **every** container's +stdout there, including Traefik, Postgres and Redis, and pushes it to +`https://loki.vione.cloud`. ### One thing to confirm with Babak Is this repo a **platform you must consume as-is**, or a **starting point you may modify**? It's in your own Gitea and nothing marks it read-only, so this roadmap -assumes you own it and will edit `config.alloy.j2`, add a dashboards provider, and -extend `group_vars`. If Babak intends it as a fixed platform, several -recommendations below change — ask before Phase 4. +assumes you own it and will edit the Alloy config, add a dashboards provider, and +add scrape jobs. If Babak intends it as a fixed platform, several +recommendations below change — ask before Phase 4. Porting it to k3s (and the two +config fixes) is already a modification worth mentioning to him. --- diff --git a/docs/roadmap/phase-11.md b/docs/roadmap/phase-11.md index d8550e2..72e702c 100644 --- a/docs/roadmap/phase-11.md +++ b/docs/roadmap/phase-11.md @@ -14,8 +14,8 @@ Babak gave you a task; deliver it like an engineer, not like a student. 1. **`README.md`** — architecture diagram, one-command start, what each container is for, and a "here's the demo" script. 2. **The five-minute demo path**, rehearsed: - - Delete every dashboard in the Grafana UI, then `ansible-playbook - playbooks/grafana-stack.yml` — the whole platform and every dashboard rebuilt + - Delete every dashboard in the Grafana UI (or the whole namespace), then + `kubectl apply -k k8s/` — the whole platform and every dashboard rebuilt from code, one command. - Submit one transaction from the frontend, then follow it through Grafana: RED dashboard → latency spike → exemplar → trace → span → logs for that span → back diff --git a/docs/roadmap/phase-2.md b/docs/roadmap/phase-2.md index 5190cd4..6cf7329 100644 --- a/docs/roadmap/phase-2.md +++ b/docs/roadmap/phase-2.md @@ -57,38 +57,55 @@ than abstract. ### Build -1. **Deploy the platform.** Provision a VM, point `inventory/hosts.yml` at it, - `ansible-galaxy collection install -r requirements.yml`, change - `grafana_admin_password` (vault it), then - `ansible-playbook playbooks/grafana-stack.yml`. Verify every endpoint from the - README's table answers: Grafana 3000, Prometheus 9090, Alertmanager 9093, - Loki 3100, Tempo 3200, Alloy UI 12345, OTLP 4317/4318. +1. **Deploy the platform.** *Deployed 2026-10-12.* The mentor's Ansible + Grafana stack was ported to the k3s cluster `falken` as `k8s/`, keeping the + same versions, configs and pipeline. Deploy with `kubectl apply -k k8s/`. + `k8s/README.md` covers deploy/teardown, credentials and every deviation + from the Ansible role. - **Where the platform lives — not on the Ledgerline server.** Grafana (3000) - and the rest come from the Ansible stack, not from Ledgerline's compose - files, so `ledgerline.vione.cloud:3000` will never show Grafana. Three - reasons it belongs on its own host: - - `czy` has 1 GB RAM and Ledgerline already fills it; the platform README - recommends 2 vCPU / 4 GB for the Grafana stack. - - Phase 1's rule is that only 22/80/443 are open on `czy`; the platform - publishes seven more ports. - - `ledgerline.vione.cloud` is behind Cloudflare's proxy, which only forwards - ports 80, 8080, 8880, 2052, 2082, 2086, 2095 (HTTP) and 443, 2053, 2083, - 2087, 2096, 8443 (HTTPS). Port 3000 (and 3100, 3200, 4318, 9090, 12345) - never reaches the origin through a proxied hostname. + **Why not the Ansible route on a VM:** + - `czy` (1 GB) can't host it. The platform README recommends 2 vCPU / 4 GB. + - `*.vione.cloud` sits behind Cloudflare's proxy, which only forwards a few + ports (80/443, 8080, 8443, …), so raw ports like `:3000` never reach any + origin. That's why `ledgerline.vione.cloud:3000` showed nothing. - So reach the platform by the obs host's IP, an SSH tunnel, or a separate - **DNS-only** (grey-cloud) record. The platform's ports are published on all - interfaces with little or no auth — firewall them to your own IP plus - `czy`'s IP (which needs 4318 for OTLP). `` below means that host, - reached from `czy` over the internet. - Then open the Alloy UI and **read the component graph** — it shows you the exact - pipeline, including the fact that the OTLP receiver has only a traces output. - Seeing that in the UI before you get bitten by it is worth ten minutes. - *(Optional laptop fallback while the VM is being set up: `grafana/otel-lgtm`, - a single container with everything pre-wired. Useful for a first trace, but - don't build on it — its pipeline differs from your real platform in exactly the - ways that matter.)* + **Endpoints:** each one is a subdomain behind Traefik. Everything except + Grafana uses basic auth (`k8s/.credentials`). + + | | | + |---|---| + | Grafana | https://grafana.vione.cloud | + | Prometheus | https://prometheus.vione.cloud | + | Alertmanager | https://alertmanager.vione.cloud | + | Loki | https://loki.vione.cloud | + | Tempo (query) | https://tempo.vione.cloud | + | Alloy UI | https://alloy.vione.cloud | + | OTLP/HTTP ingest | https://otlp.vione.cloud | + + Container logs from `czy` reach Loki through an Alloy agent there (the + `alloy-agent` service in `docker-compose.traefik.yml`). + + **Verified live:** all of the following worked, using a scratch trace sent + from `czy`. + - Every endpoint answers, and returns 401 without auth. + - Prometheus scrapes all six platform targets. + - Traces flow czy → Alloy → Tempo. + - Tempo's metrics-generator emits `traces_spanmetrics_*` and + `traces_service_graph_*` with exemplars. + - czy container logs land in Loki. + + **Two platform bugs, fixed in the port.** As shipped, the Ansible role + produced no span metrics or service graph (Tempo needs + `metrics_generator.traces_storage`), and Prometheus discarded exemplars + (missing `--enable-feature=exemplar-storage`). + + **One issue left open on purpose:** Tempo labels its exemplars `traceID`, + but the datasource's `exemplarTraceIdDestinations` expects `trace_id`. + + Now open the Alloy UI and **read the component graph**. It shows you the + exact pipeline, including the fact that the OTLP receiver has only a traces + output. Seeing that in the UI before you get bitten by it is worth ten + minutes. 2. Pick the **`sanctions`** service (the smallest) and add nothing but zero-code auto-instrumentation: ``` @@ -98,9 +115,12 @@ than abstract. and change its command to `opentelemetry-instrument uvicorn services.sanctions.app:app --host 0.0.0.0 --port 8003` with `OTEL_SERVICE_NAME=sanctions`, - `OTEL_EXPORTER_OTLP_ENDPOINT=http://:4318`, + `OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.vione.cloud`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` — pointing at **Alloy**, not at - Tempo directly. + Tempo directly — plus the basic-auth header + `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20` + (URL-encoded: the space is `%20`; see `k8s/README.md`). Keep the + credential in the gitignored `.env.observability` on czy, not in compose. 3. Fire a request. Open Grafana → Explore → Tempo → find the trace. **Done when:** you can see a single-service trace in Tempo, and you can explain out diff --git a/docs/roadmap/phase-4.md b/docs/roadmap/phase-4.md index 631b343..476d4c0 100644 --- a/docs/roadmap/phase-4.md +++ b/docs/roadmap/phase-4.md @@ -72,17 +72,20 @@ means anything). 3. Add a redaction processor with a denylist of field names. 4. Ship all seven services' logs via **stdout**, and confirm they arrive in Loki with a `container` label (Alloy's relabel rule sets it from the Docker container - name). Verify the `trace_id` field is present *and lowercase* in the JSON body. + name, e.g. `ledgerline-gateway-1`). Here that's the Alloy agent on czy + (`observability/alloy-agent/config.alloy`, already running), not the + cluster's Alloy. Verify the `trace_id` field is present *and lowercase* in + the JSON body. 5. **Prove the silent-drop for yourself.** Add an OTLP log exporter to one service, send some records, and watch nothing appear in Loki. Then check the Alloy UI: the receiver shows traffic, and the pipeline shows no logs output. Now extend - `config.alloy.j2` — add a `logs` output on the receiver plus an - `otelcol.exporter.loki` → `loki.write` chain — re-run the playbook, and watch + `k8s/alloy/config.alloy` — add a `logs` output on the receiver plus an + `otelcol.exporter.loki` → `loki.write` chain — `kubectl apply -k k8s/`, and watch them arrive. Write down what you'd have done to debug this if you *hadn't* known. That is the most transferable half-hour in the whole project. 6. Confirm the provisioned correlation works in both directions: Loki's `derivedFields` on `trace_id` → Tempo, and Tempo's `tracesToLogsV2` - (`filterByTraceID: true`) → Loki. Both are already in `datasources.yml.j2`; you + (`filterByTraceID: true`) → Loki. Both are already in `k8s/grafana/datasources.yml`; you are verifying, not writing. If a link doesn't appear, the regex didn't match your field name — check that before anything else. 6. **Migrate the test suite's log-counting helpers.** Three pathology tests diff --git a/docs/roadmap/phase-5.md b/docs/roadmap/phase-5.md index 8ee9adf..0abf5b4 100644 --- a/docs/roadmap/phase-5.md +++ b/docs/roadmap/phase-5.md @@ -62,10 +62,14 @@ first), but sequenced after Phase 4 per the roadmap's ordering. and can be enabled per-tenant — at the cost of load on the trace store. - **Exemplars.** A histogram bucket can carry a sample trace ID. That's what lets you click a spike on a latency graph and land in the exact trace. On your platform - this is already wired end to end: Tempo's generator sets `send_exemplars: true`, - and the Prometheus datasource declares `exemplarTraceIdDestinations` with - `name: trace_id`. If you emit your own histograms with exemplars, the label must - be `trace_id` to match. + this is wired: Tempo's generator sets `send_exemplars: true`, Prometheus stores + them (`--enable-feature=exemplar-storage`, added in `k8s/`; the Ansible role + lacked it), and the Prometheus datasource declares `exemplarTraceIdDestinations` + with `name: trace_id`. If you emit your own histograms with exemplars, the label + must be `trace_id` to match. **But not quite end to end:** Tempo's generator + labels its exemplars `traceID` (verified live), so the span-metric exemplars + don't match that destination. Find the fix yourself and choose one: the + datasource, Tempo, or both. - **Push vs pull.** Prometheus scrapes (`pull`); OTLP pushes. With the Collector you can do either — receive OTLP from apps, then either expose a `/metrics` endpoint for Prometheus to scrape, or remote-write into Prometheus/Mimir. Understand that @@ -92,9 +96,11 @@ first), but sequenced after Phase 4 per the roadmap's ordering. - `ledgerline.db.pool.in_use` (Observable UpDownCounter; label: `service`) 3. **Get your custom metrics somewhere they'll actually land.** Alloy drops OTLP metrics, so pick a path and implement it: - - *Platform grain:* expose `/metrics` on each service and add them to - `prometheus_extra_scrape_jobs` in `group_vars/all.yml`, then re-run the - playbook. Prometheus scrapes your services directly. Note the irony and the + - *Platform grain:* expose `/metrics` on each service and add scrape jobs to + `k8s/prometheus/prometheus.yml` (the role's `prometheus_extra_scrape_jobs`), + then `kubectl apply -k k8s/`. Prometheus scrapes your services directly. + **First decide how Prometheus on falken reaches czy**: the Traefik + override strips every service's host port (see `phase-1.md`). Note the irony and the lesson — you're back to a pull model and a second client library, which is exactly the fragmentation OTel exists to fix. - *Match the diagram:* add `prometheus.scrape` + `prometheus.remote_write` to diff --git a/docs/roadmap/phase-6.md b/docs/roadmap/phase-6.md index 8738d19..a6fd270 100644 --- a/docs/roadmap/phase-6.md +++ b/docs/roadmap/phase-6.md @@ -45,22 +45,27 @@ recreate in seconds.* Your platform provisions **datasources only** — there is no dashboards provider at all. So this phase is entirely yours, and the right home for it is inside the -Ansible role, using the same mechanism as everything else. +platform's own manifests (`k8s/grafana/`, the k3s port of the Ansible role), +using the same mechanism as everything else. -1. **Extend the role.** Add - `roles/observability/templates/grafana-stack/config/grafana/provisioning/dashboards/dashboards.yml.j2` - (a file-based dashboard provider pointing at a mounted directory), a - `config/grafana/dashboards/` directory for the JSON, and the tasks to template - and copy them. The Compose template already mounts - `./config/grafana/provisioning` read-only, so the provider is picked up on - restart. +1. **Extend `k8s/grafana/`.** Add a file-based dashboard provider (a + `provisioning/dashboards/` YAML pointing at a mounted directory) and the + dashboard JSON. Ship both through `configMapGenerator`, exactly as + `datasources.yml` is shipped today, and mount them in `deployment.yaml`. + Two k8s-specific things to work out: + - A ConfigMap is capped at about 1 MiB, and client-side `kubectl apply` + also stores a copy in an annotation. Large generated dashboards push + against that. + - The configMap hash suffix rolls the Grafana pod on every change. + Decide whether that's what you want. 2. **Generate the JSON from code**, don't hand-write it. Use the **Grafana Foundation SDK (Python)**: one function producing a RED dashboard for a given service name, looped over the seven services; a second producing a system overview (service graph, stream lag, pool saturation, error-budget burn). Commit - the generator *and* its output; the Ansible role ships the output. -3. **The demo is now `ansible-playbook`.** Delete every dashboard through the - Grafana UI, re-run `playbooks/grafana-stack.yml`, and watch them all come back. + the generator *and* its output; `k8s/grafana/` ships the output. +3. **The demo is now `kubectl apply -k k8s/`.** Delete every dashboard through + the Grafana UI, re-apply (or go further: `kubectl delete -k k8s/` and rebuild + the whole platform), and watch them all come back. That is a strictly stronger answer to Babak's "destroy and recreate in seconds" than a local Makefile target, because it's the same command that builds the platform. @@ -71,6 +76,6 @@ Ansible role, using the same mechanism as everything else. wouldn't be. 5. Add alert rules — see Phase 8 for the Alertmanager-vs-Grafana-alerting choice. -**Done when:** re-running the playbook restores every dashboard from scratch, and no +**Done when:** `kubectl apply -k k8s/` restores every dashboard from scratch, and no dashboard has ever been saved through the Grafana UI. diff --git a/docs/roadmap/phase-8.md b/docs/roadmap/phase-8.md index 069bcc5..e4861fe 100644 --- a/docs/roadmap/phase-8.md +++ b/docs/roadmap/phase-8.md @@ -73,8 +73,10 @@ come from measured numbers, not be picked from thin air). 2. Implement recording rules for the SLIs and **multi-burn-rate alert rules**. Your platform ships **Alertmanager** plus a Prometheus rules directory and the sample rules `HostExporterDown` and `HostHighCPU`, so the grain is - *Prometheus rule files → Alertmanager*, templated by the Ansible role, with - `alertmanager_webhook_url` in `group_vars` for delivery. Follow the grain — but + *Prometheus rule files → Alertmanager*. In the k3s port, rule files live in + `k8s/prometheus/rules/` (add each one to the `prometheus-rules` generator) + and delivery is a receiver in `k8s/alertmanager/alertmanager.yml` (the role's + `alertmanager_webhook_url`). Follow the grain — but be able to argue the alternative: Grafana unified alerting can evaluate across datasources (including Loki queries) and lives with your dashboards, whereas Alertmanager is metrics-only but is the ecosystem standard and gives you diff --git a/docs/roadmap/summary.md b/docs/roadmap/summary.md index 403c9c5..c71dc15 100644 --- a/docs/roadmap/summary.md +++ b/docs/roadmap/summary.md @@ -21,7 +21,7 @@ warnings), **Time-box**, **Concepts**, **Resources**, **Build**, **Done when**. | 3 | [`phase-3.md`](phase-3.md) | Not started | Distributed tracing | 2–3 evenings | Connected trace across 7 services incl. the async hop | | 4 | [`phase-4.md`](phase-4.md) | Not started | Structured logs | 1–2 evenings | Two-way trace↔log correlation | | 5 | [`phase-5.md`](phase-5.md) | Not started | Metrics | 2 evenings | RED per service, service map, exemplars | -| 6 | [`phase-6.md`](phase-6.md) | Not started | Dashboards as code | 2 evenings | `make obs-reset` | +| 6 | [`phase-6.md`](phase-6.md) | Not started | Dashboards as code | 2 evenings | `kubectl apply -k k8s/` restores all dashboards | | 7 | [`phase-7.md`](phase-7.md) | Not started | k6 + percentiles | 1–2 evenings | Baseline vs degraded load profiles | | 8 | [`phase-8.md`](phase-8.md) | Not started | SLO / error budget / RTO / RPO | 1–2 evenings | `docs/SLO.md` + burn-rate alerts | | 9 | [`phase-9.md`](phase-9.md) | Not started | Bottleneck hunting | 2 evenings | `docs/INVESTIGATION.md` | @@ -37,8 +37,9 @@ warnings), **Time-box**, **Concepts**, **Resources**, **Build**, **Done when**. Traefik and a Let's Encrypt certificate, served at a real domain. Plumbing only — no telemetry, no app changes. **Done** — see `phase-1.md`. - **2 — The model, and one signal end to end.** Learn the OTel API/SDK/Collector - split and the M.E.L.T. vocabulary; deploy the real Grafana-stack platform via - Ansible and get one auto-instrumented trace into Tempo. + split and the M.E.L.T. vocabulary; deploy the real Grafana-stack platform + (the mentor's Ansible stack, ported to k3s as `k8s/` — already deployed on + `falken`) and get one auto-instrumented trace into Tempo. - **3 — Distributed tracing.** The heart of the task: auto- and manual-instrument all seven services, and hand-propagate trace context across the Redis Stream hop, which carries none by default. @@ -50,8 +51,8 @@ warnings), **Time-box**, **Concepts**, **Resources**, **Build**, **Done when**. silently. Tempo's metrics-generator already produces span metrics and the service graph — don't duplicate it in the collector. - **6 — Dashboards and alerts as code.** Generate dashboard JSON with the - Grafana Foundation SDK and ship it through the Ansible role, so "destroy - and recreate in seconds" is a real, repeatable command. + Grafana Foundation SDK and ship it with the platform (`k8s/grafana/`), so + "destroy and recreate in seconds" is a real, repeatable command. - **7 — Load testing with k6, and percentiles.** Learn why percentiles can't be averaged, what coordinated omission is, and drive Ledgerline with arrival-rate executors to get a clean before/after load profile. diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..e42b8f6 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,117 @@ +# Observability platform on k3s (`falken`) + +The Grafana stack that Ledgerline reports to. It is a **port of the mentor's +Ansible role** (`ansible/roles/observability/templates/grafana-stack/`) to +Kubernetes: same image versions, same configs, same pipeline behaviour. The +`ansible/` directory stays as the reference it was ported from. + +Ledgerline itself still runs on `czy` with Docker Compose (see +`docs/roadmap/phase-1.md`). Only the platform lives here. + +## What's deployed + +Namespace `ledgerline-observability`. Service names match the compose service +names, so every in-config URL (`http://prometheus:9090/api/v1/write`, +`tempo:4317`, …) is unchanged from the Ansible templates. + +| Component | Image | Endpoint | Auth | +|---|---|---|---| +| Grafana | `grafana/grafana:12.1.1` | https://grafana.vione.cloud | Grafana login | +| Prometheus | `prom/prometheus:v3.5.0` | https://prometheus.vione.cloud | basic auth | +| Alertmanager | `prom/alertmanager:v0.28.1` | https://alertmanager.vione.cloud | basic auth | +| Loki (query + push) | `grafana/loki:3.5.3` | https://loki.vione.cloud | basic auth | +| Tempo (query) | `grafana/tempo:2.8.2` | https://tempo.vione.cloud | basic auth | +| Alloy UI | `grafana/alloy:v1.10.2` | https://alloy.vione.cloud | basic auth | +| OTLP/HTTP ingest (Alloy :4318) | — | https://otlp.vione.cloud | basic auth | +| node-exporter (falken host) | `prom/node-exporter:v1.9.1` | cluster-internal | — | + +OTLP/gRPC (4317) is cluster-internal only. Apps use OTLP/HTTP. + +### Differences from the Ansible stack, and why + +- **Alloy's Docker-log half runs on czy, not here.** k3s uses containerd, so + there's no Docker socket, and Ledgerline's containers are on czy anyway. The + `discovery.docker` → `loki.source.docker` chain is in + `observability/alloy-agent/config.alloy`, run by the `alloy-agent` service in + `docker-compose.traefik.yml`. It pushes to `loki.vione.cloud`. The in-cluster + Alloy keeps the OTLP receiver exactly as it was: **traces output only**. +- **Endpoints are subdomains behind Traefik + basic auth** instead of raw host + ports. `vione.cloud` sits behind Cloudflare's proxy, which only forwards + 80/443 (plus a few others), so raw ports like `:3000` would never work. +- **Resource requests/limits** on every container (falken is shared). +- **Not ported:** Prometheus app scrape jobs (`prometheus_extra_scrape_jobs` is + empty in the role too; see `prometheus/prometheus.yml`). + +The CLAUDE.md platform facts still hold: +- OTLP metrics and logs are silently dropped. +- Span metrics and the service graph come from Tempo's metrics-generator. +- The Loki derived field matches lowercase `trace_id`. +- Datasources are provisioned; dashboards are not. + +## Deploy + +```bash +# once: generate credentials (k8s/.credentials, gitignored) and create secrets. +# Also writes .env.observability for the czy agent. +./k8s/create-secrets.sh + +# deploy, or re-apply after editing any config file +kubectl apply -k k8s/ +``` + +Every config file goes through a `configMapGenerator`. Its name carries a +content hash, so editing e.g. `alloy/config.alloy` and re-applying rolls that +pod automatically. That's the equivalent of re-running the playbook. + +If the API server isn't reachable from your machine, build locally and apply +on the node: + +```bash +KUBECTL="ssh falken kubectl" ./k8s/create-secrets.sh +kubectl kustomize k8s/ | ssh falken kubectl apply -f - +``` + +**czy agent** (once, then `./deploy.sh` keeps it running): + +```bash +scp .env.observability czy:/opt/apps/LedgerLine/.env.observability +./deploy.sh +``` + +## Teardown / rebuild + +```bash +kubectl delete -k k8s/ # removes everything, including PVCs (all data) +``` + +`grafana-admin` and `obs-basic-auth` are deleted with the namespace. Run +`create-secrets.sh` again before re-applying; it reuses `k8s/.credentials`. + +## Credentials + +`k8s/.credentials` (mode 600, gitignored): +- `OBS_USER` / `OBS_PASSWORD`: basic auth for every endpoint except Grafana. +- `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD`: the Grafana login. + +```bash +source k8s/.credentials +curl -s -u "$OBS_USER:$OBS_PASSWORD" https://prometheus.vione.cloud/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}' +``` + +For an OpenTelemetry SDK (the header value must be URL-encoded, so the space +becomes `%20`): + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.vione.cloud +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20$(printf '%s:%s' "$OBS_USER" "$OBS_PASSWORD" | base64)" +``` + +## Known gaps + +- **App metrics (Phase 5):** Prometheus here can't reach czy's services. + Their host ports are stripped by the Traefik override. Decide the path + before adding scrape jobs. +- **czy host metrics:** node-exporter here watches falken. czy's own + saturation (1 vCPU / 1 GB) is invisible until a node-exporter runs there + and has a scrape path. diff --git a/k8s/alertmanager/alertmanager.yml b/k8s/alertmanager/alertmanager.yml new file mode 100644 index 0000000..7278fed --- /dev/null +++ b/k8s/alertmanager/alertmanager.yml @@ -0,0 +1,17 @@ +global: + resolve_timeout: 5m + +route: + receiver: default + group_by: [alertname, instance] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + +# No-op receiver (the Ansible role's default when alertmanager_webhook_url is +# empty). To deliver alerts, add e.g.: +# webhook_configs: +# - url: "https://..." +# send_resolved: true +receivers: + - name: default diff --git a/k8s/alertmanager/deployment.yaml b/k8s/alertmanager/deployment.yaml new file mode 100644 index 0000000..44f26cf --- /dev/null +++ b/k8s/alertmanager/deployment.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: alertmanager + template: + metadata: + labels: + app: alertmanager + spec: + containers: + - name: alertmanager + image: prom/alertmanager:v0.28.1 + args: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + ports: + - containerPort: 9093 + readinessProbe: + httpGet: + path: /-/ready + port: 9093 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + volumeMounts: + - name: config + mountPath: /etc/alertmanager + - name: data + mountPath: /alertmanager + volumes: + - name: config + configMap: + name: alertmanager-config + - name: data + persistentVolumeClaim: + claimName: alertmanager-data diff --git a/k8s/alertmanager/ingress.yaml b/k8s/alertmanager/ingress.yaml new file mode 100644 index 0000000..2450582 --- /dev/null +++ b/k8s/alertmanager/ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alertmanager + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - alertmanager.vione.cloud + secretName: alertmanager-tls + rules: + - host: alertmanager.vione.cloud + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: alertmanager + port: + number: 9093 diff --git a/k8s/alertmanager/kustomization.yaml b/k8s/alertmanager/kustomization.yaml new file mode 100644 index 0000000..7b696e1 --- /dev/null +++ b/k8s/alertmanager/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - pvc.yaml + - service.yaml + - ingress.yaml + +configMapGenerator: + - name: alertmanager-config + files: + - alertmanager.yml diff --git a/k8s/alertmanager/pvc.yaml b/k8s/alertmanager/pvc.yaml new file mode 100644 index 0000000..baed32f --- /dev/null +++ b/k8s/alertmanager/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: alertmanager-data +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/alertmanager/service.yaml b/k8s/alertmanager/service.yaml new file mode 100644 index 0000000..eacad95 --- /dev/null +++ b/k8s/alertmanager/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: alertmanager +spec: + type: ClusterIP + selector: + app: alertmanager + ports: + - name: http + port: 9093 + targetPort: 9093 diff --git a/k8s/alloy/config.alloy b/k8s/alloy/config.alloy new file mode 100644 index 0000000..5e1745e --- /dev/null +++ b/k8s/alloy/config.alloy @@ -0,0 +1,35 @@ +// Port of ansible/roles/observability/templates/grafana-stack/config/alloy/config.alloy.j2. +// +// The Ansible config also tails container logs from the local Docker socket +// (discovery.docker -> discovery.relabel -> loki.source.docker -> loki.write). +// On k3s there is no Docker socket, and Ledgerline's containers run on czy +// anyway, so that half lives in the czy-side agent: +// observability/alloy-agent/config.alloy. +// +// What remains is unchanged: the OTLP receiver wires a `traces` output and +// nothing else. OTLP metrics and logs are accepted and silently dropped. + +logging { + level = "info" +} + +otelcol.receiver.otlp "default" { + grpc { + endpoint = "0.0.0.0:4317" + } + http { + endpoint = "0.0.0.0:4318" + } + output { + traces = [otelcol.exporter.otlp.tempo.input] + } +} + +otelcol.exporter.otlp "tempo" { + client { + endpoint = "tempo:4317" + tls { + insecure = true + } + } +} diff --git a/k8s/alloy/deployment.yaml b/k8s/alloy/deployment.yaml new file mode 100644 index 0000000..7fa39d3 --- /dev/null +++ b/k8s/alloy/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alloy +spec: + replicas: 1 + selector: + matchLabels: + app: alloy + template: + metadata: + labels: + app: alloy + spec: + containers: + - name: alloy + image: grafana/alloy:v1.10.2 + args: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + ports: + - containerPort: 12345 + - containerPort: 4317 + - containerPort: 4318 + readinessProbe: + httpGet: + path: /-/ready + port: 12345 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 384Mi + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: data + mountPath: /var/lib/alloy/data + volumes: + - name: config + configMap: + name: alloy-config + - name: data + emptyDir: {} diff --git a/k8s/alloy/ingress.yaml b/k8s/alloy/ingress.yaml new file mode 100644 index 0000000..1c0785f --- /dev/null +++ b/k8s/alloy/ingress.yaml @@ -0,0 +1,42 @@ +# Two hosts, both behind basic auth: +# otlp.vione.cloud -> OTLP/HTTP ingest (:4318). Apps use +# OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.vione.cloud, +# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and an +# Authorization header. OTLP/gRPC (:4317) is +# cluster-internal only. +# alloy.vione.cloud -> Alloy UI (:12345), the component graph. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alloy + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - otlp.vione.cloud + - alloy.vione.cloud + secretName: alloy-tls + rules: + - host: otlp.vione.cloud + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: alloy + port: + number: 4318 + - host: alloy.vione.cloud + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: alloy + port: + number: 12345 diff --git a/k8s/alloy/kustomization.yaml b/k8s/alloy/kustomization.yaml new file mode 100644 index 0000000..bb6f08b --- /dev/null +++ b/k8s/alloy/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml + - ingress.yaml + +configMapGenerator: + - name: alloy-config + files: + - config.alloy diff --git a/k8s/alloy/service.yaml b/k8s/alloy/service.yaml new file mode 100644 index 0000000..90d57b7 --- /dev/null +++ b/k8s/alloy/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: alloy +spec: + type: ClusterIP + selector: + app: alloy + ports: + - name: http + port: 12345 + targetPort: 12345 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 diff --git a/k8s/auth/kustomization.yaml b/k8s/auth/kustomization.yaml new file mode 100644 index 0000000..69d7f93 --- /dev/null +++ b/k8s/auth/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - middleware.yaml diff --git a/k8s/auth/middleware.yaml b/k8s/auth/middleware.yaml new file mode 100644 index 0000000..aa7a6d5 --- /dev/null +++ b/k8s/auth/middleware.yaml @@ -0,0 +1,14 @@ +# Shared basic auth for every ingress whose backend has no auth of its own +# (Prometheus, Alertmanager, Loki, Tempo, Alloy UI, OTLP ingest). Grafana uses +# its own login instead. The htpasswd secret is created by create-secrets.sh, +# never committed. +# +# Ingresses reference it as: +# traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: basic-auth +spec: + basicAuth: + secret: obs-basic-auth diff --git a/k8s/create-secrets.sh b/k8s/create-secrets.sh new file mode 100755 index 0000000..7392f24 --- /dev/null +++ b/k8s/create-secrets.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Create the platform's secrets in the cluster. Idempotent: credentials are +# generated once into k8s/.credentials (gitignored, mode 600) and reused on +# every later run, so re-running only re-applies the same secrets. +# +# ./k8s/create-secrets.sh +# +# Creates, in namespace ledgerline-observability: +# obs-basic-auth htpasswd for the Traefik basic-auth middleware +# (Prometheus, Alertmanager, Loki, Tempo, Alloy UI, OTLP) +# grafana-admin Grafana admin user/password +# and writes .env.observability at the repo root (gitignored) for the czy-side +# Alloy log agent; copy it to the server once (see k8s/README.md). +# +# KUBECTL defaults to `kubectl`; e.g. KUBECTL="ssh falken kubectl" to run it on +# the node when the API server isn't reachable from here. +set -euo pipefail + +cd "$(dirname "$0")" +KUBECTL="${KUBECTL:-kubectl}" +NS=ledgerline-observability +CREDS=.credentials + +if [ ! -f "$CREDS" ]; then + umask 077 + cat > "$CREDS" < ../.env.observability < 85 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU utilization" + description: "CPU utilization on {{ $labels.instance }} has exceeded 85% for ten minutes." diff --git a/k8s/prometheus/service.yaml b/k8s/prometheus/service.yaml new file mode 100644 index 0000000..59da89f --- /dev/null +++ b/k8s/prometheus/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus +spec: + type: ClusterIP + selector: + app: prometheus + ports: + - name: http + port: 9090 + targetPort: 9090 diff --git a/k8s/tempo/deployment.yaml b/k8s/tempo/deployment.yaml new file mode 100644 index 0000000..b8893e5 --- /dev/null +++ b/k8s/tempo/deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + containers: + - name: tempo + image: grafana/tempo:2.8.2 + args: + - -config.file=/etc/tempo/tempo.yml + ports: + - containerPort: 3200 + - containerPort: 4317 + - containerPort: 4318 + readinessProbe: + httpGet: + path: /ready + port: 3200 + initialDelaySeconds: 15 + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + memory: 1Gi + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-config + - name: data + persistentVolumeClaim: + claimName: tempo-data diff --git a/k8s/tempo/ingress.yaml b/k8s/tempo/ingress.yaml new file mode 100644 index 0000000..d518781 --- /dev/null +++ b/k8s/tempo/ingress.yaml @@ -0,0 +1,26 @@ +# Tempo's query API (:3200) only. Apps send spans to Alloy (otlp.vione.cloud), +# never to Tempo directly. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tempo + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - tempo.vione.cloud + secretName: tempo-tls + rules: + - host: tempo.vione.cloud + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tempo + port: + number: 3200 diff --git a/k8s/tempo/kustomization.yaml b/k8s/tempo/kustomization.yaml new file mode 100644 index 0000000..e0c1646 --- /dev/null +++ b/k8s/tempo/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - pvc.yaml + - service.yaml + - ingress.yaml + +configMapGenerator: + - name: tempo-config + files: + - tempo.yml diff --git a/k8s/tempo/pvc.yaml b/k8s/tempo/pvc.yaml new file mode 100644 index 0000000..713f170 --- /dev/null +++ b/k8s/tempo/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/k8s/tempo/service.yaml b/k8s/tempo/service.yaml new file mode 100644 index 0000000..bc23573 --- /dev/null +++ b/k8s/tempo/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: tempo +spec: + type: ClusterIP + selector: + app: tempo + ports: + - name: http + port: 3200 + targetPort: 3200 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 diff --git a/k8s/tempo/tempo.yml b/k8s/tempo/tempo.yml new file mode 100644 index 0000000..1ee5f01 --- /dev/null +++ b/k8s/tempo/tempo.yml @@ -0,0 +1,47 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 336h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + # Not in the Ansible template, which is broken without it: the + # local-blocks processor needs a traces WAL, and when it can't initialise, + # Tempo 2.8 starts *no* processors for the tenant ("local blocks processor + # requires traces wal"), so service-graphs and span-metrics never emit. + traces_storage: + path: /var/tempo/generator/traces + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics, local-blocks] diff --git a/observability/alloy-agent/config.alloy b/observability/alloy-agent/config.alloy new file mode 100644 index 0000000..4509376 --- /dev/null +++ b/observability/alloy-agent/config.alloy @@ -0,0 +1,45 @@ +// Alloy log agent on czy, where Ledgerline's containers run. +// +// This is the Docker half of the platform's Alloy config +// (ansible/roles/observability/templates/grafana-stack/config/alloy/config.alloy.j2), +// moved here because the platform itself now runs on k3s (k8s/), where there is +// no Docker socket. Components are unchanged except loki.write, which pushes +// to the cluster's Loki over HTTPS with basic auth instead of http://loki:3100. +// +// OTLP is NOT received here: apps send spans to the platform's Alloy at +// https://otlp.vione.cloud (k8s/alloy/config.alloy). + +logging { + level = "info" +} + +discovery.docker "local" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "docker" { + targets = discovery.docker.local.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } +} + +loki.source.docker "local" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.docker.output + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = sys.env("LOKI_PUSH_URL") + + basic_auth { + username = sys.env("LOKI_PUSH_USER") + password = sys.env("LOKI_PUSH_PASSWORD") + } + } +}