From 2c18962243d32946ee711227dbc321bd9df7bc9f Mon Sep 17 00:00:00 2001 From: Movses Aslanyan Date: Tue, 13 Oct 2026 21:18:55 +0330 Subject: [PATCH] Label Loki streams with the Compose service name as service_name Without an explicit service_name, Loki derived it from the container label (ledgerline-sanctions-1), while spans carry service.name "sanctions". Grafana's Tempo->Loki link selects on {service_name=""}, so it matched no stream. The czy agent now sets service_name from com.docker.compose.service; `container` is unchanged. CLAUDE.md's description of the labels is updated to match. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 +++++--- observability/alloy-agent/config.alloy | 13 ++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index eb81ff1..fac8e20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -341,9 +341,11 @@ curl -s -u "$A" 'https://prometheus.vione.cloud/api/v1/status/tsdb' | jq '.data. ``` **Logs (Loki)** — confirm `trace_id` is actually in the payload, not just -in the code that was supposed to put it there. The czy agent labels streams with -the full container name (`container` and `service_name` are both e.g. -`ledgerline-gateway-1`): +in the code that was supposed to put it there. The czy agent labels each stream +with `container` (the full container name, e.g. `ledgerline-gateway-1`) and +`service_name` (the Compose service name, e.g. `gateway`, matching the spans' +`service.name` so Grafana's Tempo→Loki link selects the right stream). Streams +from before that relabel (2026-10-13) have `service_name` = the container name: ```bash curl -s -u "$A" -G 'https://loki.vione.cloud/loki/api/v1/query_range' \ --data-urlencode 'query={container="ledgerline-gateway-1"} | json | trace_id != ""' \ diff --git a/observability/alloy-agent/config.alloy b/observability/alloy-agent/config.alloy index 4509376..f3a31d8 100644 --- a/observability/alloy-agent/config.alloy +++ b/observability/alloy-agent/config.alloy @@ -4,7 +4,8 @@ // (ansible/roles/observability/templates/grafana-stack/config/alloy/config.alloy.j2), // moved here because the platform itself now runs on k3s (k8s/), where there is // no Docker socket. Components are unchanged except loki.write, which pushes -// to the cluster's Loki over HTTPS with basic auth instead of http://loki:3100. +// to the cluster's Loki over HTTPS with basic auth instead of http://loki:3100, +// and one extra relabel rule that sets service_name (see below). // // OTLP is NOT received here: apps send spans to the platform's Alloy at // https://otlp.vione.cloud (k8s/alloy/config.alloy). @@ -25,6 +26,16 @@ discovery.relabel "docker" { regex = "/(.*)" target_label = "container" } + + // Not in the Ansible config. Without it Loki derives service_name from + // `container` (ledgerline-sanctions-1), while spans carry service.name + // "sanctions" -- and Grafana's Tempo->Loki link selects on + // {service_name=""}, so it matched no stream. The Compose + // service name is the services/ directory name, i.e. the OTel service.name. + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service_name" + } } loki.source.docker "local" {