Port the observability platform to k3s and ship czy logs to it

k8s/ is now a faithful port of the mentor's Ansible Grafana stack
(Prometheus, Alertmanager, Loki, Tempo, Alloy, Grafana, node-exporter)
for the ledgerline-observability namespace on falken: same versions and
configs, one *.vione.cloud ingress per tool behind Traefik basic auth,
secrets generated by create-secrets.sh into gitignored files.

Two fixes over the Ansible role, both verified live: Tempo's
metrics-generator needs traces_storage or it runs no processors, and
Prometheus needs exemplar-storage to keep exemplars.

Ledgerline stays on czy; an Alloy agent in the Traefik override ships
its container logs to Loki. Roadmap and CLAUDE.md now point at k8s/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

k8s/prometheus/deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus:v3.5.0
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time=15d
+            - --web.enable-lifecycle
+            - --web.enable-remote-write-receiver
+            # Not in the Ansible template. Without it Prometheus discards the
+            # exemplars Tempo's metrics-generator remote-writes
+            # (send_exemplars: true), so the datasource's
+            # exemplarTraceIdDestinations has nothing to link.
+            - --enable-feature=exemplar-storage
+          ports:
+            - containerPort: 9090
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+          resources:
+            requests:
+              cpu: 50m
+              memory: 256Mi
+            limits:
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/prometheus/prometheus.yml
+              subPath: prometheus.yml
+            - name: rules
+              mountPath: /etc/prometheus/rules
+            - name: data
+              mountPath: /prometheus
+      volumes:
+        - name: config
+          configMap:
+            name: prometheus-config
+        - name: rules
+          configMap:
+            name: prometheus-rules
+        - name: data
+          persistentVolumeClaim:
+            claimName: prometheus-data

k8s/prometheus/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - prometheus.vione.cloud
+      secretName: prometheus-tls
+  rules:
+    - host: prometheus.vione.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: prometheus
+                port:
+                  number: 9090

k8s/prometheus/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - pvc.yaml
+  - service.yaml
+  - ingress.yaml
+
+configMapGenerator:
+  - name: prometheus-config
+    files:
+      - prometheus.yml
+  - name: prometheus-rules
+    files:
+      - rules/host.rules.yml

k8s/prometheus/prometheus.yml

@@ -0,0 +1,41 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["prometheus:9090"]
+
+  - job_name: node-exporter
+    static_configs:
+      - targets: ["node-exporter:9100"]
+
+  - job_name: loki
+    static_configs:
+      - targets: ["loki:3100"]
+
+  - job_name: tempo
+    static_configs:
+      - targets: ["tempo:3200"]
+
+  - job_name: alloy
+    static_configs:
+      - targets: ["alloy:12345"]
+
+  - job_name: grafana
+    static_configs:
+      - targets: ["grafana:3000"]
+
+  # Application scrape jobs go here -- the equivalent of the Ansible role's
+  # `prometheus_extra_scrape_jobs`. Ledgerline runs on czy with its service
+  # ports unpublished, so a reachable path to /metrics has to be decided first
+  # (Phase 5). Re-apply with `kubectl apply -k k8s/` after editing.

k8s/prometheus/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheus-data
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

k8s/prometheus/rules/host.rules.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: host.rules
+    rules:
+      - alert: HostExporterDown
+        expr: up{job="node-exporter"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter is unavailable"
+          description: "Prometheus has failed to scrape {{ $labels.instance }} for more than two minutes."
+
+      - alert: HostHighCPU
+        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU utilization"
+          description: "CPU utilization on {{ $labels.instance }} has exceeded 85% for ten minutes."

k8s/prometheus/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+spec:
+  type: ClusterIP
+  selector:
+    app: prometheus
+  ports:
+    - name: http
+      port: 9090
+      targetPort: 9090