Record swallowed exceptions on their spans

Where an error is caught and logged under a span that would otherwise
stay clean, also record_exception and set ERROR on that span: the
CONSUMER span when a stream message fails, fraud.score_transaction when
the sanctions call times out or fails, and the SERVER span of every
/readyz check (a 503 already made it ERROR, but without the exception).

Left alone where instrumentation already records it: ledger's app-wide
exception handler (FastAPI SERVER span), caught httpx errors (CLIENT
span), and the stream publish failure (PRODUCER span). The consumers'
failed reads run outside any span, so they stay log-only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

services/audit/consumer.py

@@ -5,7 +5,7 @@ import logging
 import sqlalchemy as sa
 from opentelemetry import propagate, trace
 from opentelemetry.instrumentation.utils import suppress_instrumentation
-from opentelemetry.trace import SpanKind
+from opentelemetry.trace import SpanKind, StatusCode
 
 from common.db import Session
 from common.redis_client import redis_client
@@ -82,9 +82,11 @@ async def run():
                         "messaging.consumer.group.name": GROUP,
                         "messaging.message.id": msg_id,
                     },
-                ):
+                ) as span:
                     try:
                         await _handle(fields)
-                    except Exception:
+                    except Exception as exc:
+                        span.record_exception(exc)
+                        span.set_status(StatusCode.ERROR, str(exc))
                         logger.exception(f"audit failed to process message {msg_id}")
                     await redis_client.xack(settings.stream_name, GROUP, msg_id)

services/fraud/app.py

@@ -3,6 +3,7 @@ from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from opentelemetry import trace
+from opentelemetry.trace import StatusCode
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 
 from common.http import downstream_client
@@ -52,6 +53,9 @@ async def readyz():
     try:
         await redis_client.ping()
     except Exception as exc:
+        span = trace.get_current_span()
+        span.record_exception(exc)
+        span.set_status(StatusCode.ERROR, str(exc))
         logger.exception(f"readyz check failed: {exc}")
         return JSONResponse(status_code=503, content={"status": "unavailable"})
     return {"status": "ok"}
@@ -81,10 +85,14 @@ async def score_transaction(body: ScoreRequest):
                     f"{settings.sanctions_url}/internal/lookup",
                     json={"holder_name": body.holder_name, "request_id": body.request_id},
                 )
-            except httpx.TimeoutException:
+            except httpx.TimeoutException as exc:
+                span.record_exception(exc)
+                span.set_status(StatusCode.ERROR, "sanctions lookup timed out")
                 logger.exception(f"sanctions lookup timed out for request_id={body.request_id}")
                 return JSONResponse(status_code=504, content={"error": "sanctions lookup timed out"})
             except httpx.RequestError as exc:
+                span.record_exception(exc)
+                span.set_status(StatusCode.ERROR, "sanctions lookup failed")
                 logger.exception(f"sanctions lookup failed for request_id={body.request_id}: {exc}")
                 return JSONResponse(status_code=502, content={"error": "sanctions lookup failed"})
 

services/insights/app.py

@@ -4,8 +4,10 @@ import time
 
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
+from opentelemetry import trace
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.trace import StatusCode
 
 from common.chaos_flags import flags_fingerprint
 from common.logging_config import configure_logging
@@ -67,6 +69,9 @@ async def readyz():
         async with engine.connect() as conn:
             await conn.exec_driver_sql("SELECT 1")
     except Exception as exc:
+        span = trace.get_current_span()
+        span.record_exception(exc)
+        span.set_status(StatusCode.ERROR, str(exc))
         logger.exception(f"readyz check failed: {exc}")
         return JSONResponse(status_code=503, content={"status": "unavailable"})
     return {"status": "ok"}

services/ledger/app.py

@@ -2,6 +2,7 @@ from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, field_validator
 from opentelemetry import trace
+from opentelemetry.trace import StatusCode
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
@@ -75,6 +76,9 @@ async def readyz():
         async with normal_engine.connect() as conn:
             await conn.exec_driver_sql("SELECT 1")
     except Exception as exc:
+        span = trace.get_current_span()
+        span.record_exception(exc)
+        span.set_status(StatusCode.ERROR, str(exc))
         logger.exception(f"readyz check failed: {exc}")
         return JSONResponse(status_code=503, content={"status": "unavailable"})
     return {"status": "ok"}

services/notifier/app.py

@@ -4,8 +4,10 @@ import asyncio
 
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
+from opentelemetry import trace
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.trace import StatusCode
 
 from common.db import engine
 from common.logging_config import configure_logging
@@ -55,6 +57,9 @@ async def readyz():
             await conn.exec_driver_sql("SELECT 1")
         await redis_client.ping()
     except Exception as exc:
+        span = trace.get_current_span()
+        span.record_exception(exc)
+        span.set_status(StatusCode.ERROR, str(exc))
         logger.exception(f"readyz check failed: {exc}")
         return JSONResponse(status_code=503, content={"status": "unavailable"})
     return {"status": "ok"}

services/notifier/consumer.py

@@ -6,7 +6,7 @@ import uuid
 import sqlalchemy as sa
 from opentelemetry import propagate, trace
 from opentelemetry.instrumentation.utils import suppress_instrumentation
-from opentelemetry.trace import SpanKind
+from opentelemetry.trace import SpanKind, StatusCode
 
 from common.db import Session
 from common.redis_client import redis_client
@@ -89,9 +89,11 @@ async def run():
                         "messaging.consumer.group.name": GROUP,
                         "messaging.message.id": msg_id,
                     },
-                ):
+                ) as span:
                     try:
                         await _handle(fields)
-                    except Exception:
+                    except Exception as exc:
+                        span.record_exception(exc)
+                        span.set_status(StatusCode.ERROR, str(exc))
                         logger.exception(f"notifier failed to process message {msg_id}")
                     await redis_client.xack(settings.stream_name, GROUP, msg_id)

services/sanctions/app.py

@@ -4,6 +4,7 @@ from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from opentelemetry import trace
+from opentelemetry.trace import StatusCode
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
@@ -50,6 +51,9 @@ async def readyz():
         async with engine.connect() as conn:
             await conn.exec_driver_sql("SELECT 1")
     except Exception as exc:
+        span = trace.get_current_span()
+        span.record_exception(exc)
+        span.set_status(StatusCode.ERROR, str(exc))
         logger.exception(f"readyz check failed: {exc}")
         return JSONResponse(status_code=503, content={"status": "unavailable"})
     return {"status": "ok"}