Port the observability platform to k3s and ship czy logs to it

k8s/ is now a faithful port of the mentor's Ansible Grafana stack
(Prometheus, Alertmanager, Loki, Tempo, Alloy, Grafana, node-exporter)
for the ledgerline-observability namespace on falken: same versions and
configs, one *.vione.cloud ingress per tool behind Traefik basic auth,
secrets generated by create-secrets.sh into gitignored files.

Two fixes over the Ansible role, both verified live: Tempo's
metrics-generator needs traces_storage or it runs no processors, and
Prometheus needs exemplar-storage to keep exemplars.

Ledgerline stays on czy; an Alloy agent in the Traefik override ships
its container logs to Loki. Roadmap and CLAUDE.md now point at k8s/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

k8s/loki/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: loki
+  template:
+    metadata:
+      labels:
+        app: loki
+    spec:
+      containers:
+        - name: loki
+          image: grafana/loki:3.5.3
+          args:
+            - -config.file=/etc/loki/loki.yml
+          ports:
+            - containerPort: 3100
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 3100
+            initialDelaySeconds: 15
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              memory: 768Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/loki
+            - name: data
+              mountPath: /var/loki
+      volumes:
+        - name: config
+          configMap:
+            name: loki-config
+        - name: data
+          persistentVolumeClaim:
+            claimName: loki-data

k8s/loki/ingress.yaml

@@ -0,0 +1,26 @@
+# Query API and push endpoint (/loki/api/v1/push) on one host, both behind
+# basic auth. The Alloy log agent on czy pushes here.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/router.middlewares: ledgerline-observability-basic-auth@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - loki.vione.cloud
+      secretName: loki-tls
+  rules:
+    - host: loki.vione.cloud
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100

k8s/loki/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - pvc.yaml
+  - service.yaml
+  - ingress.yaml
+
+configMapGenerator:
+  - name: loki-config
+    files:
+      - loki.yml

k8s/loki/loki.yml

@@ -0,0 +1,33 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /var/loki
+  replication_factor: 1
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  filesystem:
+    directory: /var/loki/chunks
+
+compactor:
+  working_directory: /var/loki/compactor
+
+limits_config:
+  retention_period: 336h
+  allow_structured_metadata: true

k8s/loki/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-data
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

k8s/loki/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+spec:
+  type: ClusterIP
+  selector:
+    app: loki
+  ports:
+    - name: http
+      port: 3100
+      targetPort: 3100