Instrument SQLAlchemy in every service with a database

sanctions, notifier, ledger, insights and audit now call setup_telemetry
and instrument their engines with one SQLAlchemyInstrumentor call per
process (a second instrument() call is ignored, leaving that engine with
connect spans only). ledger and insights pass both their normal and
starved engines. The FastAPI services also get the server span, with the
same /healthz and receive/send exclusions as sanctions.

Compose: OTEL_SEMCONV_STABILITY_OPT_IN=http,database for the stable
attribute names, the OTLP endpoint moves to the shared env, and the four
newly instrumented services get the auth header from .env.observability.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docker-compose.yml

@@ -10,6 +10,10 @@ x-app-env: &app-env
   # Merged into the Resource by Resource.create(). deploy.sh sets DEPLOY_HASH
   # to the deployed git SHA; anything else (local runs) is "dev".
   OTEL_RESOURCE_ATTRIBUTES: service.version=${DEPLOY_HASH:-dev}
+  # Emit the stable semantic-convention names (http.request.method, url.path,
+  # db.query.text, ...) instead of the instrumentations' legacy defaults.
+  OTEL_SEMCONV_STABILITY_OPT_IN: http,database
+  OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud
 
 x-app-build: &app-build
   build:
@@ -102,6 +106,8 @@ services:
     environment:
       <<: *app-env
       SERVICE_NAME: ledger
+    env_file:
+      - .env.observability
     command: uvicorn services.ledger.app:app --host 0.0.0.0 --port 8001 --reload --no-access-log
     ports:
       - "8001:8001"
@@ -141,7 +147,6 @@ services:
       - ./scripts:/app/scripts
     environment:
       <<: *app-env
-      OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud
     env_file:
       - .env.observability
     command: uvicorn services.sanctions.app:app --host 0.0.0.0 --port 8003 --reload --no-access-log
@@ -164,6 +169,8 @@ services:
     environment:
       <<: *app-env
       SERVICE_NAME: insights
+    env_file:
+      - .env.observability
     command: uvicorn services.insights.app:app --host 0.0.0.0 --port 8004 --reload --no-access-log
     ports:
       - "8004:8004"
@@ -184,6 +191,8 @@ services:
     environment:
       <<: *app-env
       SERVICE_NAME: notifier
+    env_file:
+      - .env.observability
     command: uvicorn services.notifier.app:app --host 0.0.0.0 --port 8005 --reload --no-access-log
     ports:
       - "8005:8005"
@@ -204,6 +213,8 @@ services:
     environment:
       <<: *app-env
       SERVICE_NAME: audit
+    env_file:
+      - .env.observability
     command: python -m services.audit.worker
 
 volumes:

services/audit/worker.py

@@ -1,10 +1,16 @@
 import asyncio
 
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+
+from common.db import engine
 from common.logging_config import configure_logging
+from common.telemetry import setup_telemetry
 
 from . import consumer
 
 configure_logging("audit")
+setup_telemetry(service_name="audit")
+SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine])
 
 if __name__ == "__main__":
     asyncio.run(consumer.run())

services/insights/app.py

@@ -4,22 +4,35 @@ import time
 
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 from common.chaos_flags import flags_fingerprint
 from common.logging_config import configure_logging
 from common.redis_client import redis_client
 from common.settings import settings
+from common.telemetry import setup_telemetry
 
 import sqlalchemy as sa
 
 from . import chaos
 from .analytics import account_exists, compute_summary
-from .db import NormalSession, engine, session_factory
+from .db import NormalSession, engine, normal_engine, session_factory, starved_engine
 
 SERVICE_NAME = "insights"
 logger = configure_logging(SERVICE_NAME)
+setup_telemetry(service_name=SERVICE_NAME)
 app = FastAPI(title=SERVICE_NAME)
 
+FastAPIInstrumentor.instrument_app(
+    app,
+    excluded_urls="/healthz",
+    exclude_spans=["receive", "send"],
+)
+SQLAlchemyInstrumentor().instrument(
+    engines=[normal_engine.sync_engine, starved_engine.sync_engine]
+)
+
 # Cache-stampede protection for the Redis-cached branch below: a burst of
 # concurrent requests for the same not-yet-cached (account, flags) key would
 # otherwise all pay the full computation cost at once instead of one paying

services/ledger/app.py

@@ -1,17 +1,30 @@
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, field_validator
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 from common.logging_config import configure_logging
+from common.telemetry import setup_telemetry
 
 from . import chaos
-from .db import normal_engine, session_factory
+from .db import normal_engine, session_factory, starved_engine
 from . import repository
 
 SERVICE_NAME = "ledger"
 logger = configure_logging(SERVICE_NAME)
+setup_telemetry(service_name=SERVICE_NAME)
 app = FastAPI(title=SERVICE_NAME)
 
+FastAPIInstrumentor.instrument_app(
+    app,
+    excluded_urls="/healthz",
+    exclude_spans=["receive", "send"],
+)
+SQLAlchemyInstrumentor().instrument(
+    engines=[normal_engine.sync_engine, starved_engine.sync_engine]
+)
+
 
 class CreateTransactionRequest(BaseModel):
     account_id: str

services/notifier/app.py

@@ -4,15 +4,19 @@ import asyncio
 
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 from common.db import engine
 from common.logging_config import configure_logging
 from common.redis_client import redis_client
+from common.telemetry import setup_telemetry
 
 from . import consumer
 
 SERVICE_NAME = "notifier"
 logger = configure_logging(SERVICE_NAME)
+setup_telemetry(service_name=SERVICE_NAME)
 
 
 @asynccontextmanager
@@ -24,6 +28,13 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(title=SERVICE_NAME, lifespan=lifespan)
 
+FastAPIInstrumentor.instrument_app(
+    app,
+    excluded_urls="/healthz",
+    exclude_spans=["receive", "send"],
+)
+SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine])
+
 
 @app.get("/")
 async def root():

services/sanctions/app.py

@@ -4,6 +4,7 @@ from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 
 from common.db import engine
 from common.logging_config import configure_logging
@@ -22,6 +23,7 @@ FastAPIInstrumentor.instrument_app(
     excluded_urls="/healthz",
     exclude_spans=["receive", "send"],
 )
+SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine])
 
 class LookupRequest(BaseModel):
     holder_name: str