Carry trace context across the Redis Stream by hand

Redis Streams have no headers, so the gateway injects traceparent into
the message fields inside a PRODUCER span. The notifier and audit
consumers extract it and open one CONSUMER span per message, so their
DB and XACK spans join the transaction's trace. The idle XREADGROUP poll
runs under suppress_instrumentation (it was a ~2s root trace every loop),
and audit strips the trace fields so its stored payload is unchanged.

The gateway also stops tracing its frontend's /api/health/ polling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

services/notifier/consumer.py

@@ -4,6 +4,9 @@ import random
 import uuid
 
 import sqlalchemy as sa
+from opentelemetry import propagate, trace
+from opentelemetry.instrumentation.utils import suppress_instrumentation
+from opentelemetry.trace import SpanKind
 
 from common.db import Session
 from common.redis_client import redis_client
@@ -12,6 +15,7 @@ from common.settings import settings
 from . import chaos
 
 logger = logging.getLogger("notifier")
+tracer = trace.get_tracer(__name__)
 
 GROUP = "notifier"
 CONSUMER = "notifier-1"
@@ -52,13 +56,15 @@ async def run():
     logger.info("notifier consumer started")
     while True:
         try:
-            resp = await redis_client.xreadgroup(
-                groupname=GROUP,
-                consumername=CONSUMER,
-                streams={settings.stream_name: ">"},
-                count=10,
-                block=2000,
-            )
+            # The blocking poll would otherwise be a ~2s root trace every loop.
+            with suppress_instrumentation():
+                resp = await redis_client.xreadgroup(
+                    groupname=GROUP,
+                    consumername=CONSUMER,
+                    streams={settings.stream_name: ">"},
+                    count=10,
+                    block=2000,
+                )
         except asyncio.CancelledError:
             raise
         except Exception:
@@ -71,8 +77,21 @@ async def run():
 
         for _, messages in resp:
             for msg_id, fields in messages:
-                try:
-                    await _handle(fields)
-                except Exception:
-                    logger.exception(f"notifier failed to process message {msg_id}")
-                await redis_client.xack(settings.stream_name, GROUP, msg_id)
+                # Parent is the gateway's PRODUCER span, carried in the message.
+                with tracer.start_as_current_span(
+                    f"process {settings.stream_name}",
+                    context=propagate.extract(fields),
+                    kind=SpanKind.CONSUMER,
+                    attributes={
+                        "messaging.system": "redis",
+                        "messaging.destination.name": settings.stream_name,
+                        "messaging.operation.type": "process",
+                        "messaging.consumer.group.name": GROUP,
+                        "messaging.message.id": msg_id,
+                    },
+                ):
+                    try:
+                        await _handle(fields)
+                    except Exception:
+                        logger.exception(f"notifier failed to process message {msg_id}")
+                    await redis_client.xack(settings.stream_name, GROUP, msg_id)