From 1ab425dc4408de32a5d64107f4bcc8bd6b71156c Mon Sep 17 00:00:00 2001 From: Movses Aslanyan Date: Tue, 13 Oct 2026 19:48:27 +0330 Subject: [PATCH] Record swallowed exceptions on their spans Where an error is caught and logged under a span that would otherwise stay clean, also record_exception and set ERROR on that span: the CONSUMER span when a stream message fails, fraud.score_transaction when the sanctions call times out or fails, and the SERVER span of every /readyz check (a 503 already made it ERROR, but without the exception). Left alone where instrumentation already records it: ledger's app-wide exception handler (FastAPI SERVER span), caught httpx errors (CLIENT span), and the stream publish failure (PRODUCER span). The consumers' failed reads run outside any span, so they stay log-only. Co-Authored-By: Claude Opus 5.5 --- services/audit/consumer.py | 8 +++++--- services/fraud/app.py | 10 +++++++++- services/insights/app.py | 5 +++++ services/ledger/app.py | 4 ++++ services/notifier/app.py | 5 +++++ services/notifier/consumer.py | 8 +++++--- services/sanctions/app.py | 4 ++++ 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/services/audit/consumer.py b/services/audit/consumer.py index ae96f5b..b1e1e52 100644 --- a/services/audit/consumer.py +++ b/services/audit/consumer.py @@ -5,7 +5,7 @@ import logging import sqlalchemy as sa from opentelemetry import propagate, trace from opentelemetry.instrumentation.utils import suppress_instrumentation -from opentelemetry.trace import SpanKind +from opentelemetry.trace import SpanKind, StatusCode from common.db import Session from common.redis_client import redis_client @@ -82,9 +82,11 @@ async def run(): "messaging.consumer.group.name": GROUP, "messaging.message.id": msg_id, }, - ): + ) as span: try: await _handle(fields) - except Exception: + except Exception as exc: + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"audit failed to process message {msg_id}") await redis_client.xack(settings.stream_name, GROUP, msg_id) diff --git a/services/fraud/app.py b/services/fraud/app.py index dfdbaf8..5841bd3 100644 --- a/services/fraud/app.py +++ b/services/fraud/app.py @@ -3,6 +3,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel from opentelemetry import trace +from opentelemetry.trace import StatusCode from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from common.http import downstream_client @@ -52,6 +53,9 @@ async def readyz(): try: await redis_client.ping() except Exception as exc: + span = trace.get_current_span() + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"readyz check failed: {exc}") return JSONResponse(status_code=503, content={"status": "unavailable"}) return {"status": "ok"} @@ -81,10 +85,14 @@ async def score_transaction(body: ScoreRequest): f"{settings.sanctions_url}/internal/lookup", json={"holder_name": body.holder_name, "request_id": body.request_id}, ) - except httpx.TimeoutException: + except httpx.TimeoutException as exc: + span.record_exception(exc) + span.set_status(StatusCode.ERROR, "sanctions lookup timed out") logger.exception(f"sanctions lookup timed out for request_id={body.request_id}") return JSONResponse(status_code=504, content={"error": "sanctions lookup timed out"}) except httpx.RequestError as exc: + span.record_exception(exc) + span.set_status(StatusCode.ERROR, "sanctions lookup failed") logger.exception(f"sanctions lookup failed for request_id={body.request_id}: {exc}") return JSONResponse(status_code=502, content={"error": "sanctions lookup failed"}) diff --git a/services/insights/app.py b/services/insights/app.py index 5dd0d75..9fac44d 100644 --- a/services/insights/app.py +++ b/services/insights/app.py @@ -4,8 +4,10 @@ import time from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry import trace from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry.trace import StatusCode from common.chaos_flags import flags_fingerprint from common.logging_config import configure_logging @@ -67,6 +69,9 @@ async def readyz(): async with engine.connect() as conn: await conn.exec_driver_sql("SELECT 1") except Exception as exc: + span = trace.get_current_span() + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"readyz check failed: {exc}") return JSONResponse(status_code=503, content={"status": "unavailable"}) return {"status": "ok"} diff --git a/services/ledger/app.py b/services/ledger/app.py index 9b02abb..187673a 100644 --- a/services/ledger/app.py +++ b/services/ledger/app.py @@ -2,6 +2,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel, field_validator from opentelemetry import trace +from opentelemetry.trace import StatusCode from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor @@ -75,6 +76,9 @@ async def readyz(): async with normal_engine.connect() as conn: await conn.exec_driver_sql("SELECT 1") except Exception as exc: + span = trace.get_current_span() + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"readyz check failed: {exc}") return JSONResponse(status_code=503, content={"status": "unavailable"}) return {"status": "ok"} diff --git a/services/notifier/app.py b/services/notifier/app.py index 91554ca..5013cf8 100644 --- a/services/notifier/app.py +++ b/services/notifier/app.py @@ -4,8 +4,10 @@ import asyncio from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry import trace from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry.trace import StatusCode from common.db import engine from common.logging_config import configure_logging @@ -55,6 +57,9 @@ async def readyz(): await conn.exec_driver_sql("SELECT 1") await redis_client.ping() except Exception as exc: + span = trace.get_current_span() + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"readyz check failed: {exc}") return JSONResponse(status_code=503, content={"status": "unavailable"}) return {"status": "ok"} diff --git a/services/notifier/consumer.py b/services/notifier/consumer.py index aa7aa4b..bf35299 100644 --- a/services/notifier/consumer.py +++ b/services/notifier/consumer.py @@ -6,7 +6,7 @@ import uuid import sqlalchemy as sa from opentelemetry import propagate, trace from opentelemetry.instrumentation.utils import suppress_instrumentation -from opentelemetry.trace import SpanKind +from opentelemetry.trace import SpanKind, StatusCode from common.db import Session from common.redis_client import redis_client @@ -89,9 +89,11 @@ async def run(): "messaging.consumer.group.name": GROUP, "messaging.message.id": msg_id, }, - ): + ) as span: try: await _handle(fields) - except Exception: + except Exception as exc: + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"notifier failed to process message {msg_id}") await redis_client.xack(settings.stream_name, GROUP, msg_id) diff --git a/services/sanctions/app.py b/services/sanctions/app.py index b1332b8..4d78a82 100644 --- a/services/sanctions/app.py +++ b/services/sanctions/app.py @@ -4,6 +4,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel from opentelemetry import trace +from opentelemetry.trace import StatusCode from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor @@ -50,6 +51,9 @@ async def readyz(): async with engine.connect() as conn: await conn.exec_driver_sql("SELECT 1") except Exception as exc: + span = trace.get_current_span() + span.record_exception(exc) + span.set_status(StatusCode.ERROR, str(exc)) logger.exception(f"readyz check failed: {exc}") return JSONResponse(status_code=503, content={"status": "unavailable"}) return {"status": "ok"}