Port the observability platform to k3s and ship czy logs to it

k8s/ is now a faithful port of the mentor's Ansible Grafana stack
(Prometheus, Alertmanager, Loki, Tempo, Alloy, Grafana, node-exporter)
for the ledgerline-observability namespace on falken: same versions and
configs, one *.vione.cloud ingress per tool behind Traefik basic auth,
secrets generated by create-secrets.sh into gitignored files.

Two fixes over the Ansible role, both verified live: Tempo's
metrics-generator needs traces_storage or it runs no processors, and
Prometheus needs exemplar-storage to keep exemplars.

Ledgerline stays on czy; an Alloy agent in the Traefik override ships
its container logs to Loki. Roadmap and CLAUDE.md now point at k8s/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

k8s/tempo/deployment.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tempo
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: tempo
+  template:
+    metadata:
+      labels:
+        app: tempo
+    spec:
+      containers:
+        - name: tempo
+          image: grafana/tempo:2.8.2
+          args:
+            - -config.file=/etc/tempo/tempo.yml
+          ports:
+            - containerPort: 3200
+            - containerPort: 4317
+            - containerPort: 4318
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 3200
+            initialDelaySeconds: 15
+          resources:
+            requests:
+              cpu: 50m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/tempo
+            - name: data
+              mountPath: /var/tempo
+      volumes:
+        - name: config
+          configMap:
+            name: tempo-config
+        - name: data
+          persistentVolumeClaim:
+            claimName: tempo-data

k8s/tempo/ingress.yaml

@@ -0,0 +1,26 @@
+# Tempo's query API (:3200) only. Apps send spans to Alloy (otlp.vione.cloud),
+# never to Tempo directly.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: tempo
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - tempo.vione.cloud
+      secretName: tempo-tls
+  rules:
+    - host: tempo.vione.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: tempo
+                port:
+                  number: 3200

k8s/tempo/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - pvc.yaml
+  - service.yaml
+  - ingress.yaml
+
+configMapGenerator:
+  - name: tempo-config
+    files:
+      - tempo.yml

k8s/tempo/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: tempo-data
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

k8s/tempo/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: tempo
+spec:
+  type: ClusterIP
+  selector:
+    app: tempo
+  ports:
+    - name: http
+      port: 3200
+      targetPort: 3200
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318

k8s/tempo/tempo.yml

@@ -0,0 +1,47 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 336h
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/blocks
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  # Not in the Ansible template, which is broken without it: the
+  # local-blocks processor needs a traces WAL, and when it can't initialise,
+  # Tempo 2.8 starts *no* processors for the tenant ("local blocks processor
+  # requires traces wal"), so service-graphs and span-metrics never emit.
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks]