"""Observable (polled) instruments shared by more than one service. Their callbacks run on prometheus_client's scrape thread, not on the event loop, so they only do thread-safe reads: a pool's checked-out count, and a query through a separate synchronous Redis client (the shared asyncio one belongs to the loop).""" import redis from opentelemetry import metrics from opentelemetry.instrumentation.utils import suppress_instrumentation from opentelemetry.metrics import CallbackOptions, Observation from sqlalchemy.ext.asyncio import AsyncEngine from .settings import settings meter = metrics.get_meter(__name__) def observe_db_pools(pools: dict[str, AsyncEngine]) -> None: """ledgerline.db.pool.in_use, one series per named pool. The SQLAlchemy instrumentation's own db.client.connections.usage names a pool by its connection string, which a service's normal and starved engines share, so it sums the two into one series.""" def callback(_options: CallbackOptions): return [ Observation(engine.sync_engine.pool.checkedout(), {"pool": name}) for name, engine in pools.items() ] meter.create_observable_up_down_counter( "ledgerline.db.pool.in_use", callbacks=[callback], unit="{connection}", description="Connections currently checked out of the pool", ) def observe_stream_lag(consumer_group: str) -> None: """ledgerline.stream.lag for this process's consumer group: stream entries not yet delivered to the group (XINFO GROUPS `lag`, Redis 7+).""" client = redis.Redis.from_url(settings.redis_url, decode_responses=True, socket_timeout=1) def callback(_options: CallbackOptions): # Untraced, or every scrape would be a one-span root trace in Tempo. with suppress_instrumentation(): groups = client.xinfo_groups(settings.stream_name) return [ Observation(group["lag"], {"consumer_group": consumer_group}) for group in groups if group["name"] == consumer_group and group["lag"] is not None ] meter.create_observable_gauge( "ledgerline.stream.lag", callbacks=[callback], unit="{message}", description="Stream entries not yet delivered to the consumer group", )