Port the observability platform to k3s and ship czy logs to it

k8s/ is now a faithful port of the mentor's Ansible Grafana stack
(Prometheus, Alertmanager, Loki, Tempo, Alloy, Grafana, node-exporter)
for the ledgerline-observability namespace on falken: same versions and
configs, one *.vione.cloud ingress per tool behind Traefik basic auth,
secrets generated by create-secrets.sh into gitignored files.

Two fixes over the Ansible role, both verified live: Tempo's
metrics-generator needs traces_storage or it runs no processors, and
Prometheus needs exemplar-storage to keep exemplars.

Ledgerline stays on czy; an Alloy agent in the Traefik override ships
its container logs to Loki. Roadmap and CLAUDE.md now point at k8s/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

k8s/alloy/config.alloy

@@ -0,0 +1,35 @@
+// Port of ansible/roles/observability/templates/grafana-stack/config/alloy/config.alloy.j2.
+//
+// The Ansible config also tails container logs from the local Docker socket
+// (discovery.docker -> discovery.relabel -> loki.source.docker -> loki.write).
+// On k3s there is no Docker socket, and Ledgerline's containers run on czy
+// anyway, so that half lives in the czy-side agent:
+// observability/alloy-agent/config.alloy.
+//
+// What remains is unchanged: the OTLP receiver wires a `traces` output and
+// nothing else. OTLP metrics and logs are accepted and silently dropped.
+
+logging {
+  level = "info"
+}
+
+otelcol.receiver.otlp "default" {
+  grpc {
+    endpoint = "0.0.0.0:4317"
+  }
+  http {
+    endpoint = "0.0.0.0:4318"
+  }
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input]
+  }
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+    tls {
+      insecure = true
+    }
+  }
+}

k8s/alloy/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alloy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: alloy
+  template:
+    metadata:
+      labels:
+        app: alloy
+    spec:
+      containers:
+        - name: alloy
+          image: grafana/alloy:v1.10.2
+          args:
+            - run
+            - --server.http.listen-addr=0.0.0.0:12345
+            - --storage.path=/var/lib/alloy/data
+            - /etc/alloy/config.alloy
+          ports:
+            - containerPort: 12345
+            - containerPort: 4317
+            - containerPort: 4318
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 12345
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 384Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/alloy
+            - name: data
+              mountPath: /var/lib/alloy/data
+      volumes:
+        - name: config
+          configMap:
+            name: alloy-config
+        - name: data
+          emptyDir: {}

k8s/alloy/ingress.yaml

@@ -0,0 +1,42 @@
+# Two hosts, both behind basic auth:
+#   otlp.vione.cloud  -> OTLP/HTTP ingest (:4318). Apps use
+#                        OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.vione.cloud,
+#                        OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf and an
+#                        Authorization header. OTLP/gRPC (:4317) is
+#                        cluster-internal only.
+#   alloy.vione.cloud -> Alloy UI (:12345), the component graph.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: alloy
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - otlp.vione.cloud
+        - alloy.vione.cloud
+      secretName: alloy-tls
+  rules:
+    - host: otlp.vione.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: alloy
+                port:
+                  number: 4318
+    - host: alloy.vione.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: alloy
+                port:
+                  number: 12345

k8s/alloy/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+
+configMapGenerator:
+  - name: alloy-config
+    files:
+      - config.alloy

k8s/alloy/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alloy
+spec:
+  type: ClusterIP
+  selector:
+    app: alloy
+  ports:
+    - name: http
+      port: 12345
+      targetPort: 12345
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318