diff --git a/docker-compose.yml b/docker-compose.yml index 0e74c49..c0ee3bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,10 @@ x-app-env: &app-env # Merged into the Resource by Resource.create(). deploy.sh sets DEPLOY_HASH # to the deployed git SHA; anything else (local runs) is "dev". OTEL_RESOURCE_ATTRIBUTES: service.version=${DEPLOY_HASH:-dev} + # Emit the stable semantic-convention names (http.request.method, url.path, + # db.query.text, ...) instead of the instrumentations' legacy defaults. + OTEL_SEMCONV_STABILITY_OPT_IN: http,database + OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud x-app-build: &app-build build: @@ -102,6 +106,8 @@ services: environment: <<: *app-env SERVICE_NAME: ledger + env_file: + - .env.observability command: uvicorn services.ledger.app:app --host 0.0.0.0 --port 8001 --reload --no-access-log ports: - "8001:8001" @@ -141,7 +147,6 @@ services: - ./scripts:/app/scripts environment: <<: *app-env - OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.vione.cloud env_file: - .env.observability command: uvicorn services.sanctions.app:app --host 0.0.0.0 --port 8003 --reload --no-access-log @@ -164,6 +169,8 @@ services: environment: <<: *app-env SERVICE_NAME: insights + env_file: + - .env.observability command: uvicorn services.insights.app:app --host 0.0.0.0 --port 8004 --reload --no-access-log ports: - "8004:8004" @@ -184,6 +191,8 @@ services: environment: <<: *app-env SERVICE_NAME: notifier + env_file: + - .env.observability command: uvicorn services.notifier.app:app --host 0.0.0.0 --port 8005 --reload --no-access-log ports: - "8005:8005" @@ -204,6 +213,8 @@ services: environment: <<: *app-env SERVICE_NAME: audit + env_file: + - .env.observability command: python -m services.audit.worker volumes: diff --git a/services/audit/worker.py b/services/audit/worker.py index 20c8575..61d7321 100644 --- a/services/audit/worker.py +++ b/services/audit/worker.py @@ -1,10 +1,16 @@ import asyncio +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor + +from common.db import engine from common.logging_config import configure_logging +from common.telemetry import setup_telemetry from . import consumer configure_logging("audit") +setup_telemetry(service_name="audit") +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) if __name__ == "__main__": asyncio.run(consumer.run()) diff --git a/services/insights/app.py b/services/insights/app.py index 06aa60c..5dd0d75 100644 --- a/services/insights/app.py +++ b/services/insights/app.py @@ -4,22 +4,35 @@ import time from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.chaos_flags import flags_fingerprint from common.logging_config import configure_logging from common.redis_client import redis_client from common.settings import settings +from common.telemetry import setup_telemetry import sqlalchemy as sa from . import chaos from .analytics import account_exists, compute_summary -from .db import NormalSession, engine, session_factory +from .db import NormalSession, engine, normal_engine, session_factory, starved_engine SERVICE_NAME = "insights" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) app = FastAPI(title=SERVICE_NAME) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument( + engines=[normal_engine.sync_engine, starved_engine.sync_engine] +) + # Cache-stampede protection for the Redis-cached branch below: a burst of # concurrent requests for the same not-yet-cached (account, flags) key would # otherwise all pay the full computation cost at once instead of one paying diff --git a/services/ledger/app.py b/services/ledger/app.py index 58b9da9..0c89e05 100644 --- a/services/ledger/app.py +++ b/services/ledger/app.py @@ -1,17 +1,30 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel, field_validator +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.logging_config import configure_logging +from common.telemetry import setup_telemetry from . import chaos -from .db import normal_engine, session_factory +from .db import normal_engine, session_factory, starved_engine from . import repository SERVICE_NAME = "ledger" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) app = FastAPI(title=SERVICE_NAME) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument( + engines=[normal_engine.sync_engine, starved_engine.sync_engine] +) + class CreateTransactionRequest(BaseModel): account_id: str diff --git a/services/notifier/app.py b/services/notifier/app.py index 63eaee8..91554ca 100644 --- a/services/notifier/app.py +++ b/services/notifier/app.py @@ -4,15 +4,19 @@ import asyncio from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.db import engine from common.logging_config import configure_logging from common.redis_client import redis_client +from common.telemetry import setup_telemetry from . import consumer SERVICE_NAME = "notifier" logger = configure_logging(SERVICE_NAME) +setup_telemetry(service_name=SERVICE_NAME) @asynccontextmanager @@ -24,6 +28,13 @@ async def lifespan(app: FastAPI): app = FastAPI(title=SERVICE_NAME, lifespan=lifespan) +FastAPIInstrumentor.instrument_app( + app, + excluded_urls="/healthz", + exclude_spans=["receive", "send"], +) +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) + @app.get("/") async def root(): diff --git a/services/sanctions/app.py b/services/sanctions/app.py index 4407a4b..723539a 100644 --- a/services/sanctions/app.py +++ b/services/sanctions/app.py @@ -4,6 +4,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from pydantic import BaseModel from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from common.db import engine from common.logging_config import configure_logging @@ -22,6 +23,7 @@ FastAPIInstrumentor.instrument_app( excluded_urls="/healthz", exclude_spans=["receive", "send"], ) +SQLAlchemyInstrumentor().instrument(engines=[engine.sync_engine]) class LookupRequest(BaseModel): holder_name: str