From ec6ddb6a2639bd7b9182a4c2fafc4977b860f1ab Mon Sep 17 00:00:00 2001 From: Movses Aslanyan Date: Tue, 13 Oct 2026 10:09:57 +0330 Subject: [PATCH] Instrument SQLAlchemy in every service with a database sanctions, notifier, ledger, insights and audit now call setup_telemetry and instrument their engines with one SQLAlchemyInstrumentor call per process (a second instrument() call is ignored, leaving that engine with connect spans only). ledger and insights pass both their normal and starved engines. The FastAPI services also get the server span, with the same /healthz and receive/send exclusions as sanctions. Compose: OTEL_SEMCONV_STABILITY_OPT_IN=http,database for the stable attribute names, the OTLP endpoint moves to the shared env, and the four newly instrumented services get the auth header from .env.observability. Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 13 ++++++++++++- services/audit/worker.py | 6 ++++++ services/insights/app.py | 15 ++++++++++++++- services/ledger/app.py | 15 ++++++++++++++- services/notifier/app.py | 11 +++++++++++ services/sanctions/app.py | 2 ++ 6 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e74c49..c0ee3bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,10 @@ x-app-env: &app-env # Merged into the Resource by Resource.create(). deploy.sh sets DEPLOY_HASH # to the deployed git SHA; anything else (local runs) is "dev". OTEL_RESOURCE_ATTRIBUTES: service.version=${DEPLOY_HASH:-dev} + # Emit the stable semantic-convention names (http.request.method, url.path, + # db.query.text, ...) instead of the instrumentations' legacy defaults. + OTEL_SEMCONV_STABILITY_OPT_IN: http,database + OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud x-app-build: &app-build build: @@ -102,6 +106,8 @@ services: environment: <<: *app-env SERVICE_NAME: ledger + env_file: + - .env.observability command: uvicorn services.ledger.app:app --host 0.0.0.0 --port 8001 --reload --no-access-log ports: - "8001:8001" @@ -141,7 +147,6 @@ services: - ./scripts:/app/scripts environment: <<: *app-env - OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud env_file: - .env.observability command: uvicorn services.sanctions.app:app --host 0.0.0.0 --port 8003 --reload --no-access-log @@ -164,6 +169,8 @@ services: environment: <<: *app-env SERVICE_NAME: insights + env_file: + - .env.observability command: uvicorn services.insights.app:app --host 0.0.0.0 --port 8004 --reload --no-access-log ports: - "8004:8004" @@ -184,6 +191,8 @@ services: environment: <<: *app-env SERVICE_NAME: notifier + env_file: + - .env.observability command: uvicorn services.notifier.app:app --host 0.0.0.0 --port 8005 --reload --no-access-log ports: - "8005:8005" @@ -204,6 +213,8 @@ services: environment: <<: *app-env SERVICE_NAME: audit + env_file: + - .env.observability command: python -m services.audit.worker volumes: diff --git a/services/audit/worker.py b/services/audit/worker.py index 20c8575..61d7321 100644 --- a/services/audit/worker.py +++ b/services/audit/worker.py @@ -1,10 +1,16 @@ import asyncio +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor + +from common.db import engine from common.logging_config import configure_logging +from common.telemetry import setup_telemetry from . import consumer configure_logging("audit") +setup_telemetry(service_name="audit") +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) if __name__ == "__main__": asyncio.run(consumer.run()) diff --git a/services/insights/app.py b/services/insights/app.py index 06aa60c..5dd0d75 100644 --- a/services/insights/app.py +++ b/services/insights/app.py @@ -4,22 +4,35 @@ import time from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.chaos_flags import flags_fingerprint from common.logging_config import configure_logging from common.redis_client import redis_client from common.settings import settings +from common.telemetry import setup_telemetry import sqlalchemy as sa from . import chaos from .analytics import account_exists, compute_summary -from .db import NormalSession, engine, session_factory +from .db import NormalSession, engine, normal_engine, session_factory, starved_engine SERVICE_NAME = "insights" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) app = FastAPI(title=SERVICE_NAME) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument( + engines=[normal_engine.sync_engine, starved_engine.sync_engine] +) + # Cache-stampede protection for the Redis-cached branch below: a burst of # concurrent requests for the same not-yet-cached (account, flags) key would # otherwise all pay the full computation cost at once instead of one paying diff --git a/services/ledger/app.py b/services/ledger/app.py index 58b9da9..0c89e05 100644 --- a/services/ledger/app.py +++ b/services/ledger/app.py @@ -1,17 +1,30 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel, field_validator +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.logging_config import configure_logging +from common.telemetry import setup_telemetry from . import chaos -from .db import normal_engine, session_factory +from .db import normal_engine, session_factory, starved_engine from . import repository SERVICE_NAME = "ledger" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) app = FastAPI(title=SERVICE_NAME) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument( + engines=[normal_engine.sync_engine, starved_engine.sync_engine] +) + class CreateTransactionRequest(BaseModel): account_id: str diff --git a/services/notifier/app.py b/services/notifier/app.py index 63eaee8..91554ca 100644 --- a/services/notifier/app.py +++ b/services/notifier/app.py @@ -4,15 +4,19 @@ import asyncio from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.db import engine from common.logging_config import configure_logging from common.redis_client import redis_client +from common.telemetry import setup_telemetry from . import consumer SERVICE_NAME = "notifier" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) @asynccontextmanager @@ -24,6 +28,13 @@ async def lifespan(app: FastAPI): app = FastAPI(title=SERVICE_NAME, lifespan=lifespan) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) + @app.get("/") async def root(): diff --git a/services/sanctions/app.py b/services/sanctions/app.py index 4407a4b..723539a 100644 --- a/services/sanctions/app.py +++ b/services/sanctions/app.py @@ -4,6 +4,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.db import engine from common.logging_config import configure_logging @@ -22,6 +23,7 @@ FastAPIInstrumentor.instrument_app( excluded_urls="/healthz", exclude_spans=["receive", "send"], ) +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) class LookupRequest(BaseModel): holder_name: str