Implement Phase 0b base project against the frozen test contract

All seven services (gateway, ledger, fraud, sanctions, insights, notifier,
audit), db/schema.sql plus optional_indexes.sql, scripts/seed.py and
smoke.py, the gateway-served frontend, and docker-compose.yml. Plain-text
stdlib logging and a weak manual X-Request-Id only -- no observability yet,
by design, per test_no_observability.py. Implements all 13 named chaos-flag
pathologies from docs/PATHOLOGIES.md behind Redis-backed runtime flags.

make smoke and make test-contract pass; make test-baseline and
make test-pathology pass except for a handful of load-sensitive tests
documented in tests/README.md rather than papered over.
This commit is contained in:
Movses Aslanyan
2026-09-05 08:12:30 +03:30
parent ccf1783779
commit 91afea92a1
51 changed files with 2757 additions and 0 deletions
+14
View File
@@ -0,0 +1,14 @@
# Copy to .env if you want to override defaults for a local, non-Compose run.
# Inside Compose, these are set directly in docker-compose.yml per service.
POSTGRES_DSN=postgresql+asyncpg://ledgerline:ledgerline@localhost:5432/ledgerline
REDIS_URL=redis://localhost:6379/0
GATEWAY_URL=http://localhost:8000
LEDGER_URL=http://localhost:8001
FRAUD_URL=http://localhost:8002
SANCTIONS_URL=http://localhost:8003
INSIGHTS_URL=http://localhost:8004
NOTIFIER_URL=http://localhost:8005
SEED=42
+22
View File
@@ -0,0 +1,22 @@
FROM python:3.12-slim
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY common ./common
COPY services ./services
COPY db ./db
COPY scripts ./scripts
# CMD differs per service -- see docker-compose.yml.
CMD ["python", "-c", "print('no default command: set one in docker-compose.yml')"]
+70
View File
@@ -0,0 +1,70 @@
# Ledgerline -- application targets (phase-0-base-project-spec.md §12) plus
# the Phase 0a test-suite targets. `make up && make seed` are prerequisites
# for every test target below.
PYTEST ?= pytest
COMPOSE ?= docker compose
SEED_DSN ?= postgresql://ledgerline:ledgerline@postgres:5432/ledgerline
GATEWAY_URL ?= http://localhost:8000
.PHONY: up down logs seed reset smoke \
chaos-clean chaos-light chaos-nightmare chaos-random \
fix-index drop-index \
test test-contract test-baseline test-pathology test-slow test-all
up:
$(COMPOSE) up -d --build
down:
$(COMPOSE) down
logs:
$(COMPOSE) logs -f
seed:
cat db/schema.sql | $(COMPOSE) exec -T postgres psql -U ledgerline -d ledgerline
$(COMPOSE) run --rm -e POSTGRES_DSN=$(SEED_DSN) gateway python scripts/seed.py
reset:
$(COMPOSE) down -v
$(MAKE) up
$(MAKE) seed
smoke:
$(COMPOSE) run --rm -e POSTGRES_DSN=$(SEED_DSN) -e GATEWAY_URL=http://gateway:8000 gateway python scripts/smoke.py
chaos-clean:
curl -sf -X POST $(GATEWAY_URL)/api/admin/chaos/profile -H 'Content-Type: application/json' -d '{"profile":"clean"}'
chaos-light:
curl -sf -X POST $(GATEWAY_URL)/api/admin/chaos/profile -H 'Content-Type: application/json' -d '{"profile":"light"}'
chaos-nightmare:
curl -sf -X POST $(GATEWAY_URL)/api/admin/chaos/profile -H 'Content-Type: application/json' -d '{"profile":"nightmare"}'
chaos-random:
curl -sf -X POST $(GATEWAY_URL)/api/admin/chaos/profile -H 'Content-Type: application/json' -d '{"profile":"random"}'
fix-index:
cat db/optional_indexes.sql | $(COMPOSE) exec -T postgres psql -U ledgerline -d ledgerline
drop-index:
cat db/drop_indexes.sql | $(COMPOSE) exec -T postgres psql -U ledgerline -d ledgerline
test:
$(PYTEST) -m "not slow"
test-contract:
$(PYTEST) -m "contract or seed or guard"
test-baseline:
$(PYTEST) -m baseline
test-pathology:
$(PYTEST) -m "pathology and not slow"
test-slow:
$(PYTEST) -m slow
test-all:
$(PYTEST)
View File
+75
View File
@@ -0,0 +1,75 @@
"""Shared low-level access to the `chaos:flags` Redis hash.
This is infrastructure (a Redis hash reader/writer), not pathology behaviour --
the mechanism each flag triggers lives in that service's own chaos.py, per
phase-0-base-project-spec.md §7 ("each pathology lives in a clearly named
chaos.py module inside its owning service"). Every service re-reads the hash
on every request; nothing here is cached in-process.
"""
import random
import time
from .redis_client import redis_client
FLAGS_KEY = "chaos:flags"
ALL_FLAGS = [
"slow_sanctions",
"sanctions_errors",
"n_plus_one",
"pool_starvation",
"row_lock_contention",
"blocking_event_loop",
"retry_storm",
"no_batching",
"unbounded_cache",
"slow_consumer",
"cold_cache",
"poison_data",
"chatty_health",
]
NAMED_PROFILES = {
"clean": set(),
"light": {"slow_sanctions", "cold_cache"},
"nightmare": set(ALL_FLAGS),
}
async def get_flags() -> dict:
raw = await redis_client.hgetall(FLAGS_KEY)
return {flag: raw.get(flag) == "1" for flag in ALL_FLAGS}
async def is_enabled(flag: str) -> bool:
return await redis_client.hget(FLAGS_KEY, flag) == "1"
async def set_flag(flag: str, enabled: bool) -> None:
if flag not in ALL_FLAGS:
raise ValueError(f"unknown flag {flag!r}")
await redis_client.hset(FLAGS_KEY, flag, "1" if enabled else "0")
async def set_profile(profile: str) -> dict:
if profile == "random":
# Seed derived from time.time(), never logged -- the active set lives
# only in the `chaos:flags` hash, per §7's "random" profile note.
rng = random.Random(time.time())
chosen = set(rng.sample(ALL_FLAGS, k=rng.randint(3, 6)))
elif profile in NAMED_PROFILES:
chosen = NAMED_PROFILES[profile]
else:
raise ValueError(f"unknown profile {profile!r}")
mapping = {flag: ("1" if flag in chosen else "0") for flag in ALL_FLAGS}
await redis_client.hset(FLAGS_KEY, mapping=mapping)
return await get_flags()
async def flags_fingerprint() -> str:
"""A short, stable string identifying the current flag combination. Used
to key the insights summary cache so a live flag change (G5) is never
masked by a stale cache entry written under a different chaos state."""
flags = await get_flags()
return "".join("1" if flags[f] else "0" for f in ALL_FLAGS)
+15
View File
@@ -0,0 +1,15 @@
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from .settings import settings
def make_engine(**overrides):
"""A fresh async engine. Services that need more than one pool profile
(see ledger/chaos.py for pool_starvation) call this more than once."""
kwargs = dict(pool_pre_ping=True)
kwargs.update(overrides)
return create_async_engine(settings.postgres_dsn, **kwargs)
engine = make_engine(pool_size=20, max_overflow=10)
Session = async_sessionmaker(engine, expire_on_commit=False)
+13
View File
@@ -0,0 +1,13 @@
import httpx
from .settings import settings
def downstream_client(request_id: str) -> httpx.AsyncClient:
"""An httpx.AsyncClient preconfigured with the 3s downstream timeout and
the X-Request-Id header forwarded, per phase-0-base-project-spec.md §9 --
this header is the only correlation mechanism that exists in Phase 0."""
return httpx.AsyncClient(
timeout=settings.downstream_timeout_seconds,
headers={"X-Request-Id": request_id},
)
+12
View File
@@ -0,0 +1,12 @@
import logging
def configure_logging(service_name: str) -> logging.Logger:
"""Plain-text, human-prose logging -- the deliberately weak Phase 0
baseline. Per phase-0-base-project-spec.md §9: no JSON, no key-value, no
structured fields. Do not "improve" this; later phases replace it."""
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
return logging.getLogger(service_name)
+5
View File
@@ -0,0 +1,5 @@
import redis.asyncio as redis
from .settings import settings
redis_client = redis.from_url(settings.redis_url, decode_responses=True)
+27
View File
@@ -0,0 +1,27 @@
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
service_name: str = "unknown"
postgres_dsn: str = "postgresql+asyncpg://ledgerline:ledgerline@postgres:5432/ledgerline"
redis_url: str = "redis://redis:6379/0"
gateway_url: str = "http://gateway:8000"
ledger_url: str = "http://ledger:8001"
fraud_url: str = "http://fraud:8002"
sanctions_url: str = "http://sanctions:8003"
insights_url: str = "http://insights:8004"
notifier_url: str = "http://notifier:8005"
downstream_timeout_seconds: float = 3.0
stream_name: str = "stream:transactions"
summary_cache_ttl_seconds: int = 900
summary_cache_ttl_seconds_cold: int = 60
settings = Settings()
+4
View File
@@ -0,0 +1,4 @@
-- Applied only by `make drop-index`. Reverses db/optional_indexes.sql.
DROP INDEX IF EXISTS transactions_account_created_idx;
DROP INDEX IF EXISTS transactions_category_idx;
+5
View File
@@ -0,0 +1,5 @@
-- Applied only by `make fix-index` (docs/phase-0-base-project-spec.md §5, §12).
-- These are the indexes deliberately omitted from db/schema.sql.
CREATE INDEX IF NOT EXISTS transactions_account_created_idx ON transactions (account_id, created_at);
CREATE INDEX IF NOT EXISTS transactions_category_idx ON transactions (category_id);
+80
View File
@@ -0,0 +1,80 @@
-- Ledgerline schema. Idempotent: safe to run against an existing database.
-- Index policy is deliberate -- see docs/phase-0-base-project-spec.md §5.
-- Do NOT add transactions(account_id, created_at) or transactions(category_id)
-- here; those live in db/optional_indexes.sql, applied only by `make fix-index`.
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
CREATE TABLE IF NOT EXISTS accounts (
id UUID PRIMARY KEY,
holder_name TEXT NOT NULL,
currency CHAR(3) NOT NULL,
balance_minor BIGINT NOT NULL,
is_hot BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS merchant_categories (
id SMALLINT PRIMARY KEY,
code TEXT NOT NULL UNIQUE,
label TEXT NOT NULL,
risk_weight NUMERIC(3,2) NOT NULL
);
CREATE TABLE IF NOT EXISTS transactions (
id UUID PRIMARY KEY,
account_id UUID NOT NULL,
kind TEXT NOT NULL,
amount_minor BIGINT NOT NULL,
currency CHAR(3),
merchant_name TEXT,
category_id SMALLINT,
status TEXT NOT NULL,
risk_score NUMERIC(4,3),
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Parallel workers otherwise let a plain sequential scan of this table
-- compete with an index scan on raw CPU parallelism for higher-selectivity
-- accounts (e.g. the 5 hot ones, each ~6-8% of all rows) -- masking exactly
-- the missing-index pathology this table's index policy (README §5) exists
-- to demonstrate. Scoped to this one table, not a database-wide setting.
ALTER TABLE transactions SET (parallel_workers = 0);
CREATE TABLE IF NOT EXISTS sanctions_entries (
id SERIAL PRIMARY KEY,
normalized_name TEXT NOT NULL,
country CHAR(2),
list_source TEXT,
added_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS notifications (
id UUID PRIMARY KEY,
transaction_id UUID NOT NULL,
channel TEXT NOT NULL,
status TEXT NOT NULL,
attempts INT NOT NULL DEFAULT 0,
sent_at TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS audit_events (
id BIGSERIAL PRIMARY KEY,
transaction_id UUID NOT NULL,
event_type TEXT NOT NULL,
payload JSONB,
recorded_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Not part of the base spec's domain model: a single-row table the seed
-- script writes its summary manifest into, so both `scripts/smoke.py` and
-- the test suite can read back what a given seed run actually produced
-- (row counts, hot/sanctioned/poison account ids, the SEED value used)
-- without re-deriving it via ad-hoc queries.
CREATE TABLE IF NOT EXISTS seed_manifest (
id SMALLINT PRIMARY KEY DEFAULT 1,
data JSONB NOT NULL
);
CREATE INDEX IF NOT EXISTS accounts_holder_name_idx ON accounts (holder_name);
CREATE INDEX IF NOT EXISTS sanctions_entries_normalized_name_idx ON sanctions_entries (normalized_name);
+205
View File
@@ -0,0 +1,205 @@
x-app-env: &app-env
POSTGRES_DSN: postgresql+asyncpg://ledgerline:ledgerline@postgres:5432/ledgerline
REDIS_URL: redis://redis:6379/0
GATEWAY_URL: http://gateway:8000
LEDGER_URL: http://ledger:8001
FRAUD_URL: http://fraud:8002
SANCTIONS_URL: http://sanctions:8003
INSIGHTS_URL: http://insights:8004
NOTIFIER_URL: http://notifier:8005
x-app-build: &app-build
build:
context: .
dockerfile: Dockerfile
x-app-volumes: &app-volumes
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
x-app-depends: &app-depends
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
services:
postgres:
image: postgres:16-alpine
# max_connections=300: five separate services each keep their own
# SQLAlchemy pool (several sized up to pool_size=20+max_overflow=10),
# all against this one Postgres instance -- comfortably clears the
# default 100 under concurrent load instead of silently rationing
# connections in a way that looks like an unrelated pathology.
command:
- postgres
- -c
- shared_preload_libraries=pg_stat_statements
- -c
- pg_stat_statements.track=all
- -c
- pg_stat_statements.max=10000
- -c
- max_connections=300
environment:
POSTGRES_USER: ledgerline
POSTGRES_PASSWORD: ledgerline
POSTGRES_DB: ledgerline
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ledgerline -d ledgerline"]
interval: 3s
timeout: 3s
retries: 20
volumes:
- pgdata:/var/lib/postgresql/data
redis:
image: redis:7-alpine
ports:
- "6379:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 20
gateway:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: gateway
command: uvicorn services.gateway.app:app --host 0.0.0.0 --port 8000 --reload --no-access-log
ports:
- "8000:8000"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
ledger:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: ledger
command: uvicorn services.ledger.app:app --host 0.0.0.0 --port 8001 --reload --no-access-log
ports:
- "8001:8001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8001/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
fraud:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: fraud
command: uvicorn services.fraud.app:app --host 0.0.0.0 --port 8002 --reload --no-access-log
ports:
- "8002:8002"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8002/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
sanctions:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: sanctions
command: uvicorn services.sanctions.app:app --host 0.0.0.0 --port 8003 --reload --no-access-log
ports:
- "8003:8003"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8003/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
insights:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: insights
command: uvicorn services.insights.app:app --host 0.0.0.0 --port 8004 --reload --no-access-log
ports:
- "8004:8004"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8004/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
notifier:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: notifier
command: uvicorn services.notifier.app:app --host 0.0.0.0 --port 8005 --reload --no-access-log
ports:
- "8005:8005"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8005/healthz"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
audit:
<<: [*app-build, *app-depends]
volumes:
- ./common:/app/common
- ./services:/app/services
- ./db:/app/db
- ./scripts:/app/scripts
environment:
<<: *app-env
SERVICE_NAME: audit
command: python -m services.audit.worker
volumes:
pgdata:
View File
+10
View File
@@ -0,0 +1,10 @@
fastapi==0.115.6
uvicorn[standard]==0.34.0
httpx==0.28.1
sqlalchemy[asyncio]==2.0.36
asyncpg==0.30.0
psycopg[binary]==3.2.3
redis==5.2.1
pydantic==2.10.4
pydantic-settings==2.7.1
python-multipart==0.0.20
+375
View File
@@ -0,0 +1,375 @@
#!/usr/bin/env python3
"""Deterministic seed data for Ledgerline -- phase-0-base-project-spec.md §8.
Idempotent: truncates then inserts. Deterministic given SEED (default 42).
Uses COPY for the transactions table so 250k rows load in well under 60s.
"""
import json
import os
import random
import time
import uuid
from datetime import datetime, timedelta, timezone
import psycopg
SEED = int(os.environ.get("SEED", "42"))
DSN = os.environ.get("POSTGRES_DSN", "postgresql://ledgerline:ledgerline@localhost:5432/ledgerline")
N_ACCOUNTS = 800
N_HOT = 5
N_CATEGORIES = 45
N_TRANSACTIONS = 250_000
N_SANCTIONS = 12_000
WINDOW_DAYS = 120
N_POISON_ACCOUNTS = 25
N_SANCTIONED_MATCHES = 40
MALFORMED_COUNTS = {
"null_currency": 1200,
"negative_deposit": 1000,
"non_ascii_merchant": 1000,
"future_dated": 1000,
"orphan_category": 800,
}
N_MALFORMED = sum(MALFORMED_COUNTS.values())
N_NORMAL = N_TRANSACTIONS - N_MALFORMED
FIRST_NAMES = [
"Amara", "Bilal", "Chiara", "Dimitri", "Elin", "Farrukh", "Greta", "Haruto",
"Ines", "Jonas", "Kalani", "Lucia", "Mateus", "Naledi", "Omar", "Priya",
"Quentin", "Rosalind", "Sami", "Tuuli", "Umberto", "Vera", "Wojciech", "Ximena",
"Yara", "Zoltan", "Anika", "Bram", "Carys", "Dovid", "Esperanza", "Felix",
"Guadalupe", "Henrik", "Ilse", "Jarrah", "Katarzyna", "Leandro", "Mireille",
"Nikolai",
]
LAST_NAMES = [
"Adeyemi", "Bergstrom", "Castellano", "Dubois", "Eriksson", "Fitzgerald",
"Gallagher", "Haddad", "Ivanov", "Jarvinen", "Kowalski", "Lindqvist",
"Moreau", "Nakamura", "Okafor", "Petrov", "Quiroga", "Rasmussen", "Sokolova",
"Tremblay", "Ueda", "Vasiliev", "Weiss", "Xu", "Yilmaz", "Zimmermann",
"Andersson", "Bianchi", "Costa", "Duarte",
]
MERCHANTS = [
"Northwind Grocers", "Blue Anchor Cafe", "Solstice Electronics", "Ferro Hardware",
"Willow Creek Pharmacy", "Ember & Oak", "Pinnacle Fitness", "Cobblestone Books",
"Lantern Street Diner", "Cedarline Motors", "Harborview Market", "Quill & Ink Supply",
"Meridian Travel", "Golden Hour Bakery", "Ironwood Furniture", "Tidal Wave Surf Shop",
"Maple & Vine Wines", "Copper Kettle Tea Co", "Driftwood Outfitters", "Verdant Nursery",
]
CURRENCY_WEIGHTS = [("EUR", 0.70), ("USD", 0.20), ("GBP", 0.10)]
def weighted_choice(rng, pairs):
values, weights = zip(*pairs)
return rng.choices(values, weights=weights, k=1)[0]
def gen_holder_names(rng, n):
combos = [f"{f} {l}" for f in FIRST_NAMES for l in LAST_NAMES]
rng.shuffle(combos)
if n > len(combos):
raise ValueError("not enough name combinations for the requested account count")
return combos[:n]
def gen_accounts(rng):
names = gen_holder_names(rng, N_ACCOUNTS)
hot_indexes = set(rng.sample(range(N_ACCOUNTS), N_HOT))
now = datetime.now(timezone.utc)
rows = []
for i, name in enumerate(names):
account_id = str(uuid.uuid4())
currency = weighted_choice(rng, CURRENCY_WEIGHTS)
balance = int(rng.lognormvariate(_median_to_mu(250_000), 0.7))
created_at = now - timedelta(days=rng.uniform(150, 400))
rows.append(
{
"id": account_id,
"holder_name": name,
"currency": currency,
"balance_minor": balance,
"is_hot": i in hot_indexes,
"created_at": created_at,
}
)
return rows
def _median_to_mu(median):
import math
return math.log(median)
def gen_categories(rng):
rows = []
for i in range(1, N_CATEGORIES + 1):
rows.append(
{
"id": i,
"code": f"cat_{i:02d}",
"label": f"Category {i:02d}",
"risk_weight": round(0.01 + (i - 1) / (N_CATEGORIES - 1) * 0.94, 2),
}
)
return rows
def account_weights(rng, accounts):
"""5 hot accounts own ~30% of rows; the remaining 795 share the rest on a
Zipf-ish distribution."""
hot = [a for a in accounts if a["is_hot"]]
cold = [a for a in accounts if not a["is_hot"]]
hot_weight_each = 0.30 / len(hot)
ranks = list(range(1, len(cold) + 1))
rng.shuffle(ranks)
zipf_raw = [1.0 / (r ** 1.1) for r in ranks]
zipf_total = sum(zipf_raw)
cold_weights = [0.70 * w / zipf_total for w in zipf_raw]
ids = [a["id"] for a in hot] + [a["id"] for a in cold]
weights = [hot_weight_each] * len(hot) + cold_weights
return ids, weights
def sample_created_at(rng, now, window_days=WINDOW_DAYS):
day_offset = rng.uniform(0, window_days)
day = now - timedelta(days=day_offset)
hour = min(23, max(0, int(rng.gauss(13, 3.5))))
minute, second = rng.randint(0, 59), rng.randint(0, 59)
dt = day.replace(hour=hour, minute=minute, second=second, microsecond=0)
if dt.weekday() >= 5 and rng.random() < 0.5:
dt -= timedelta(days=2)
return dt
def gen_normal_transactions(rng, accounts, now):
ids, weights = account_weights(rng, accounts)
statuses = [("settled", 0.92), ("rejected", 0.05), ("pending", 0.03)]
kinds = [("deposit", 0.2), ("withdrawal", 0.2), ("purchase", 0.6)]
rows = []
for _ in range(N_NORMAL):
account_id = rng.choices(ids, weights=weights, k=1)[0]
kind = weighted_choice(rng, kinds)
status = weighted_choice(rng, statuses)
amount = max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9)))
risk_score = round(rng.uniform(0, 1), 3) if status != "pending" else None
rows.append(
(
str(uuid.uuid4()),
account_id,
kind,
amount,
weighted_choice(rng, CURRENCY_WEIGHTS),
rng.choice(MERCHANTS),
rng.randint(1, N_CATEGORIES),
status,
risk_score,
sample_created_at(rng, now),
)
)
return rows
def gen_malformed_transactions(rng, accounts, poison_account_ids, now):
all_ids = [a["id"] for a in accounts if a["id"] not in poison_account_ids]
rows = []
poison_tx_ids = {}
# null_currency: every poison account gets at least one; the rest spread randomly.
remaining = MALFORMED_COUNTS["null_currency"] - len(poison_account_ids)
null_currency_accounts = list(poison_account_ids) + [
rng.choice(all_ids) for _ in range(remaining)
]
for account_id in null_currency_accounts:
tx_id = str(uuid.uuid4())
if account_id in poison_account_ids:
poison_tx_ids[account_id] = tx_id
rows.append(
(
tx_id, account_id, "purchase",
max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9))),
None, rng.choice(MERCHANTS), rng.randint(1, N_CATEGORIES),
"settled", round(rng.uniform(0, 1), 3), sample_created_at(rng, now),
)
)
for _ in range(MALFORMED_COUNTS["negative_deposit"]):
rows.append(
(
str(uuid.uuid4()), rng.choice(all_ids), "deposit",
-max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9))),
weighted_choice(rng, CURRENCY_WEIGHTS), rng.choice(MERCHANTS),
rng.randint(1, N_CATEGORIES), "settled", round(rng.uniform(0, 1), 3),
sample_created_at(rng, now),
)
)
for _ in range(MALFORMED_COUNTS["non_ascii_merchant"]):
merchant = f"{rng.choice(MERCHANTS)} \U0001F600\\x00"
rows.append(
(
str(uuid.uuid4()), rng.choice(all_ids), "purchase",
max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9))),
weighted_choice(rng, CURRENCY_WEIGHTS), merchant,
rng.randint(1, N_CATEGORIES), "settled", round(rng.uniform(0, 1), 3),
sample_created_at(rng, now),
)
)
for _ in range(MALFORMED_COUNTS["future_dated"]):
future = now + timedelta(days=3, hours=rng.uniform(-2, 2))
rows.append(
(
str(uuid.uuid4()), rng.choice(all_ids), "purchase",
max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9))),
weighted_choice(rng, CURRENCY_WEIGHTS), rng.choice(MERCHANTS),
rng.randint(1, N_CATEGORIES), "pending", None, future,
)
)
for _ in range(MALFORMED_COUNTS["orphan_category"]):
rows.append(
(
str(uuid.uuid4()), rng.choice(all_ids), "purchase",
max(1, int(rng.lognormvariate(_median_to_mu(5_000), 0.9))),
weighted_choice(rng, CURRENCY_WEIGHTS), rng.choice(MERCHANTS),
9999, "settled", round(rng.uniform(0, 1), 3), sample_created_at(rng, now),
)
)
return rows, poison_tx_ids
def gen_sanctions_entries(rng, sanctioned_holder_names):
rows = []
for name in sanctioned_holder_names:
rows.append((name, rng.choice(["US", "GB", "FR", "DE", "IR", "RU"]), "OFAC-like-list"))
fake_first = FIRST_NAMES + ["Zephyrine", "Cassius", "Odalys", "Thaddeus", "Wilhelmina"]
fake_last = LAST_NAMES + ["Voskresenskaya", "Almacita", "Brennholt", "Cortavez"]
used = set(sanctioned_holder_names)
for _ in range(N_SANCTIONS - len(sanctioned_holder_names)):
while True:
name = f"{rng.choice(fake_first)} {rng.choice(fake_last)} {rng.randint(1000,9999)}"
if name not in used:
used.add(name)
break
rows.append((name, rng.choice(["US", "GB", "FR", "DE", "IR", "RU"]), "OFAC-like-list"))
return rows
def main():
rng = random.Random(SEED)
now = datetime.now(timezone.utc)
t0 = time.monotonic()
accounts = gen_accounts(rng)
hot_accounts = [a for a in accounts if a["is_hot"]]
cold_accounts = [a for a in accounts if not a["is_hot"]]
role_pool = list(cold_accounts)
rng.shuffle(role_pool)
poison_accounts = role_pool[:N_POISON_ACCOUNTS]
sanctioned_accounts = role_pool[N_POISON_ACCOUNTS : N_POISON_ACCOUNTS + N_SANCTIONED_MATCHES]
clean_account = role_pool[N_POISON_ACCOUNTS + N_SANCTIONED_MATCHES]
cold_sample = role_pool[N_POISON_ACCOUNTS + N_SANCTIONED_MATCHES + 1 : N_POISON_ACCOUNTS + N_SANCTIONED_MATCHES + 61]
poison_account_ids = [a["id"] for a in poison_accounts]
sanctioned_account_ids = [a["id"] for a in sanctioned_accounts]
hot_account_ids = [a["id"] for a in hot_accounts]
cold_account_id_sample = [a["id"] for a in cold_sample]
clean_account_id = clean_account["id"]
categories = gen_categories(rng)
normal_txs = gen_normal_transactions(rng, accounts, now)
malformed_txs, poison_tx_ids = gen_malformed_transactions(rng, accounts, set(poison_account_ids), now)
all_txs = normal_txs + malformed_txs
rng.shuffle(all_txs)
sanctions_entries = gen_sanctions_entries(rng, [a["holder_name"] for a in sanctioned_accounts])
print(f"generated {len(accounts)} accounts, {len(all_txs)} transactions, "
f"{len(categories)} categories, {len(sanctions_entries)} sanctions entries "
f"in {time.monotonic() - t0:.1f}s (SEED={SEED})")
with psycopg.connect(DSN, autocommit=False) as conn:
with conn.cursor() as cur:
cur.execute(
"TRUNCATE accounts, merchant_categories, transactions, sanctions_entries, "
"notifications, audit_events RESTART IDENTITY"
)
with cur.copy(
"COPY accounts (id, holder_name, currency, balance_minor, is_hot, created_at) FROM STDIN"
) as copy:
for a in accounts:
copy.write_row((a["id"], a["holder_name"], a["currency"], a["balance_minor"], a["is_hot"], a["created_at"]))
with cur.copy(
"COPY merchant_categories (id, code, label, risk_weight) FROM STDIN"
) as copy:
for c in categories:
copy.write_row((c["id"], c["code"], c["label"], c["risk_weight"]))
with cur.copy(
"COPY transactions (id, account_id, kind, amount_minor, currency, merchant_name, "
"category_id, status, risk_score, created_at) FROM STDIN"
) as copy:
for row in all_txs:
copy.write_row(row)
with cur.copy(
"COPY sanctions_entries (normalized_name, country, list_source) FROM STDIN"
) as copy:
for row in sanctions_entries:
copy.write_row(row)
cur.execute("ANALYZE accounts, transactions, merchant_categories, sanctions_entries")
manifest = {
"seed": SEED,
"generated_at": now.isoformat(),
"counts": {
"accounts": len(accounts),
"transactions": len(all_txs),
"merchant_categories": len(categories),
"sanctions_entries": len(sanctions_entries),
},
"hot_account_ids": hot_account_ids,
"poison_account_ids": poison_account_ids,
"sanctioned_account_ids": sanctioned_account_ids,
"clean_account_id": clean_account_id,
"cold_account_ids_sample": cold_account_id_sample,
"malformed_row_count": len(malformed_txs),
}
cur.execute(
"INSERT INTO seed_manifest (id, data) VALUES (1, %s::jsonb) "
"ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data",
(json.dumps(manifest),),
)
conn.commit()
if poison_tx_ids:
import redis as redis_sync
redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
r = redis_sync.from_url(redis_url, decode_responses=True)
r.delete("poison:tx_by_account")
r.hset("poison:tx_by_account", mapping=poison_tx_ids)
elapsed = time.monotonic() - t0
print(f"seed complete in {elapsed:.1f}s")
print("row counts:")
for table, count in manifest["counts"].items():
print(f" {table}: {count}")
print(f"hot account ids: {hot_account_ids}")
print(f"poison account ids: {poison_account_ids[:5]}... ({len(poison_account_ids)} total)")
print(f"malformed rows: {manifest['malformed_row_count']}")
print(f"clean_account_id: {clean_account_id}")
if __name__ == "__main__":
main()
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""One transaction, one insights call, assert 200s -- phase-0-base-project-spec.md §12."""
import json
import os
import sys
import urllib.request
import psycopg
GATEWAY_URL = os.environ.get("GATEWAY_URL", "http://localhost:8000")
DSN = os.environ.get("POSTGRES_DSN", "postgresql://ledgerline:ledgerline@localhost:5432/ledgerline")
def _get(url):
with urllib.request.urlopen(url, timeout=10) as resp:
return resp.status, json.loads(resp.read())
def _post(url, payload):
data = json.dumps(payload).encode()
req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}, method="POST")
try:
with urllib.request.urlopen(req, timeout=10) as resp:
return resp.status, json.loads(resp.read())
except urllib.error.HTTPError as exc:
return exc.code, json.loads(exc.read())
def main():
with psycopg.connect(DSN, autocommit=True) as conn:
with conn.cursor() as cur:
cur.execute("SELECT data FROM seed_manifest LIMIT 1")
row = cur.fetchone()
if not row:
print("smoke: no seed manifest found -- run `make seed` first", file=sys.stderr)
sys.exit(1)
account_id = row[0]["clean_account_id"]
status, body = _post(
f"{GATEWAY_URL}/api/transactions",
{
"account_id": account_id,
"kind": "deposit",
"amount_minor": 500,
"currency": "EUR",
"merchant_name": "Smoke Test Merchant",
},
)
assert status == 201, f"transaction failed: {status} {body}"
print(f"transaction ok: {body['transaction_id']} status={body['status']}")
status, body = _get(f"{GATEWAY_URL}/api/accounts/{account_id}/insights")
assert status == 200, f"insights failed: {status} {body}"
print(f"insights ok: {body}")
print("smoke test passed")
if __name__ == "__main__":
main()
View File
View File
+68
View File
@@ -0,0 +1,68 @@
import asyncio
import json
import logging
import sqlalchemy as sa
from common.db import Session
from common.redis_client import redis_client
from common.settings import settings
logger = logging.getLogger("audit")
GROUP = "audit"
CONSUMER = "audit-1"
async def ensure_group():
try:
await redis_client.xgroup_create(settings.stream_name, GROUP, id="$", mkstream=True)
except Exception as exc:
if "BUSYGROUP" not in str(exc):
raise
async def _handle(fields: dict):
async with Session() as session:
await session.execute(
sa.text(
"INSERT INTO audit_events (transaction_id, event_type, payload, recorded_at) "
"VALUES (:tx_id, 'transaction_processed', CAST(:payload AS jsonb), now())"
),
{"tx_id": fields.get("transaction_id"), "payload": json.dumps(fields)},
)
await session.commit()
async def run():
"""Second, independent consumer group on the same stream as `notifier` --
proves the async fork fans out to genuinely separate consumers, not one
consumer doing both jobs."""
await ensure_group()
logger.info("audit consumer started")
while True:
try:
resp = await redis_client.xreadgroup(
groupname=GROUP,
consumername=CONSUMER,
streams={settings.stream_name: ">"},
count=10,
block=2000,
)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("audit consumer read failed")
await asyncio.sleep(1)
continue
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
try:
await _handle(fields)
except Exception:
logger.exception(f"audit failed to process message {msg_id}")
await redis_client.xack(settings.stream_name, GROUP, msg_id)
+10
View File
@@ -0,0 +1,10 @@
import asyncio
from common.logging_config import configure_logging
from . import consumer
configure_logging("audit")
if __name__ == "__main__":
asyncio.run(consumer.run())
View File
+77
View File
@@ -0,0 +1,77 @@
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from common.http import downstream_client
from common.logging_config import configure_logging
from common.redis_client import redis_client
from common.settings import settings
from . import chaos
from .scoring import score
SERVICE_NAME = "fraud"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
class ScoreRequest(BaseModel):
account_id: str
holder_name: str
category_id: int | None = None
amount_minor: int
request_id: str = "unknown"
@app.get("/")
async def root():
return {"service": SERVICE_NAME, "version": "0.1.0"}
@app.get("/healthz")
async def healthz(request: Request):
if request.headers.get("X-Ledgerline-Probe") == "chatty-health":
logger.info("received chatty-health healthz probe from gateway")
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
try:
await redis_client.ping()
except Exception as exc:
logger.exception(f"readyz check failed: {exc}")
return JSONResponse(status_code=503, content={"status": "unavailable"})
return {"status": "ok"}
@app.post("/internal/score")
async def score_transaction(body: ScoreRequest):
logger.info(f"scoring account {body.account_id} request_id={body.request_id}")
await redis_client.incr("fraud:calls_total")
if await chaos.blocking_event_loop_enabled():
logger.info(f"blocking_event_loop chaos engaged for request_id={body.request_id}")
chaos.synchronous_cpu_work()
async with downstream_client(body.request_id) as client:
try:
resp = await client.post(
f"{settings.sanctions_url}/internal/lookup",
json={"holder_name": body.holder_name, "request_id": body.request_id},
)
except httpx.TimeoutException:
logger.exception(f"sanctions lookup timed out for request_id={body.request_id}")
return JSONResponse(status_code=504, content={"error": "sanctions lookup timed out"})
except httpx.RequestError as exc:
logger.exception(f"sanctions lookup failed for request_id={body.request_id}: {exc}")
return JSONResponse(status_code=502, content={"error": "sanctions lookup failed"})
if resp.status_code >= 300:
logger.info(f"sanctions returned {resp.status_code} for request_id={body.request_id}")
return JSONResponse(status_code=502, content={"error": "sanctions lookup failed"})
matched = resp.json().get("matched", False)
risk_score = score(body.category_id, matched)
return {"risk_score": risk_score, "matched": matched}
+22
View File
@@ -0,0 +1,22 @@
"""blocking_event_loop -- phase-0-base-project-spec.md §7. Owned by `fraud`."""
import hashlib
import time
from common.chaos_flags import is_enabled
_CPU_WORK_SECONDS = 0.15
async def blocking_event_loop_enabled() -> bool:
return await is_enabled("blocking_event_loop")
def synchronous_cpu_work() -> None:
"""~120ms of real, synchronous, CPU-bound work run directly inside an
`async def` handler -- no run_in_executor/to_thread. This is the point:
it blocks the whole event loop, serialising every other in-flight
request on this worker for the duration."""
deadline = time.perf_counter() + _CPU_WORK_SECONDS
data = b"ledgerline-fraud-scoring"
while time.perf_counter() < deadline:
data = hashlib.sha256(data).digest()
+18
View File
@@ -0,0 +1,18 @@
import random
def category_weight(category_id: int | None) -> float:
"""A fake but deterministic-ish stand-in for merchant_categories.risk_weight
(fraud has no direct DB access -- Redis only, per phase-0-base-project-spec.md §4).
The business logic does not need to be correct, only to produce a plausible
spread of scores."""
if category_id is None:
return 0.3
return 0.01 + (category_id % 45) / 44 * 0.94
def score(category_id: int | None, matched: bool) -> float:
base = category_weight(category_id) * random.uniform(0.05, 0.3)
if matched:
return min(1.0, base + random.uniform(0.55, 0.9))
return min(1.0, base)
View File
+151
View File
@@ -0,0 +1,151 @@
import asyncio
import uuid
from pathlib import Path
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, field_validator
from common.chaos_flags import ALL_FLAGS, get_flags, set_flag, set_profile
from common.logging_config import configure_logging
from common.settings import settings
from . import orchestrator
from .orchestrator import UpstreamFailure
SERVICE_NAME = "gateway"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
STATIC_DIR = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
# A shared, connection-pooling client for the simple read-only proxy routes
# below -- opening a fresh httpx.AsyncClient (and TCP connection) per request
# is real, needless overhead under concurrent load. orchestrator.py's
# downstream_client() stays per-request since it carries per-transaction
# headers (X-Request-Id).
_proxy_client = httpx.AsyncClient(
timeout=settings.downstream_timeout_seconds,
limits=httpx.Limits(max_connections=100, max_keepalive_connections=50),
)
DOWNSTREAM_URLS = {
"ledger": settings.ledger_url,
"fraud": settings.fraud_url,
"sanctions": settings.sanctions_url,
"insights": settings.insights_url,
"notifier": settings.notifier_url,
}
class TransactionRequest(BaseModel):
account_id: str
kind: str
amount_minor: int
currency: str
merchant_name: str
category_id: int | None = None
@field_validator("kind")
@classmethod
def kind_must_be_known(cls, v):
if v not in ("deposit", "withdrawal", "purchase"):
raise ValueError("kind must be one of deposit, withdrawal, purchase")
return v
@field_validator("amount_minor")
@classmethod
def deposit_amount_must_be_non_negative(cls, v, info):
if info.data.get("kind") == "deposit" and v < 0:
raise ValueError("amount_minor must be non-negative for a deposit")
return v
class ChaosFlagRequest(BaseModel):
flag: str
enabled: bool
class ChaosProfileRequest(BaseModel):
profile: str
@app.get("/", include_in_schema=False)
async def index():
"""Serves the frontend (phase-0-base-project-spec.md §10). This overrides
§4's generic `GET / -> {"service", "version"}` info route, which the
other five HTTP services still expose as-is."""
return FileResponse(STATIC_DIR / "index.html")
@app.get("/healthz")
async def healthz():
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
return {"status": "ok"}
@app.get("/api/accounts")
async def list_accounts():
resp = await _proxy_client.get(f"{settings.ledger_url}/internal/accounts")
return JSONResponse(status_code=resp.status_code, content=resp.json())
@app.get("/api/accounts/{account_id}/insights")
async def account_insights(account_id: str):
resp = await _proxy_client.get(f"{settings.insights_url}/internal/accounts/{account_id}/summary")
return JSONResponse(status_code=resp.status_code, content=resp.json())
@app.post("/api/transactions", status_code=201)
async def submit_transaction(body: TransactionRequest):
request_id = str(uuid.uuid4())
try:
result = await orchestrator.create_transaction(body.model_dump(), request_id)
except UpstreamFailure as exc:
return JSONResponse(
status_code=exc.status_code, content={"error": exc.message, "request_id": request_id}
)
return JSONResponse(status_code=201, content=result)
@app.get("/api/health/all")
async def health_all():
async def check(name, url):
try:
resp = await _proxy_client.get(f"{url}/healthz", timeout=2.0)
return name, "ok" if resp.status_code == 200 else "down"
except httpx.RequestError:
return name, "down"
results = await asyncio.gather(*[check(name, url) for name, url in DOWNSTREAM_URLS.items()])
services = {"gateway": "ok", **dict(results)}
return {"services": services}
@app.get("/api/admin/chaos")
async def get_chaos():
return {"flags": await get_flags()}
@app.post("/api/admin/chaos")
async def post_chaos(body: ChaosFlagRequest):
if body.flag not in ALL_FLAGS:
return JSONResponse(status_code=400, content={"error": f"unknown flag {body.flag!r}"})
await set_flag(body.flag, body.enabled)
return {"flags": await get_flags()}
@app.post("/api/admin/chaos/profile")
async def post_chaos_profile(body: ChaosProfileRequest):
try:
flags = await set_profile(body.profile)
except ValueError as exc:
return JSONResponse(status_code=400, content={"error": str(exc)})
return {"flags": flags}
+23
View File
@@ -0,0 +1,23 @@
"""retry_storm, no_batching, chatty_health -- phase-0-base-project-spec.md §7.
All three are owned by `gateway`."""
import random
from common.chaos_flags import is_enabled
DOWNSTREAMS = ("ledger", "fraud", "sanctions", "insights", "notifier")
async def retry_storm_enabled() -> bool:
return await is_enabled("retry_storm")
async def no_batching_enabled() -> bool:
return await is_enabled("no_batching")
async def chatty_health_enabled() -> bool:
return await is_enabled("chatty_health")
def synthetic_line_item_count() -> int:
return random.randint(1, 5)
+167
View File
@@ -0,0 +1,167 @@
import logging
import time
import uuid
import httpx
from common.http import downstream_client
from common.redis_client import redis_client
from common.settings import settings
from . import chaos
logger = logging.getLogger("gateway")
REJECT_THRESHOLD = 0.8
class UpstreamFailure(Exception):
def __init__(self, status_code: int, message: str):
self.status_code = status_code
self.message = message
async def _post(client: httpx.AsyncClient, url: str, json: dict) -> httpx.Response:
try:
return await client.post(url, json=json)
except httpx.TimeoutException as exc:
raise UpstreamFailure(504, f"{url} timed out") from exc
except httpx.RequestError as exc:
raise UpstreamFailure(502, f"{url} unreachable") from exc
async def _score_once(client: httpx.AsyncClient, payload: dict) -> dict:
resp = await _post(client, f"{settings.fraud_url}/internal/score", payload)
if resp.status_code >= 300:
status = resp.status_code if resp.status_code in (502, 504) else 502
raise UpstreamFailure(status, "fraud scoring failed")
return resp.json()
async def _score_with_retry(client: httpx.AsyncClient, payload: dict, retry_storm: bool) -> dict:
"""retry_storm: on any non-2xx from fraud, retry 3 times immediately --
no backoff, no jitter, no circuit breaker (phase-0-base-project-spec.md §7).
"No circuit breaker" is taken literally here: once triggered, it fires all
3 retries unconditionally rather than stopping early on the first success
-- it never checks whether it still needs to. The last attempt's outcome
wins."""
attempts = 4 if retry_storm else 1
result = None
last_exc = None
for _ in range(attempts):
try:
result = await _score_once(client, payload)
last_exc = None
except UpstreamFailure as exc:
last_exc = exc
if last_exc is not None:
raise last_exc
return result
async def _run_chatty_health_probe(client: httpx.AsyncClient) -> None:
"""chatty_health: GET /healthz on all five downstreams, sequentially,
before doing any real work."""
headers = {"X-Ledgerline-Probe": "chatty-health"}
for name in chaos.DOWNSTREAMS:
url = getattr(settings, f"{name}_url")
try:
await client.get(f"{url}/healthz", headers=headers, timeout=2.0)
except httpx.RequestError:
pass
async def create_transaction(payload: dict, request_id: str) -> dict:
start = time.perf_counter()
logger.info(f"start transaction for account {payload['account_id']} request_id={request_id}")
retry_storm = await chaos.retry_storm_enabled()
no_batching = await chaos.no_batching_enabled()
chatty_health = await chaos.chatty_health_enabled()
async with downstream_client(request_id) as client:
if chatty_health:
await _run_chatty_health_probe(client)
create_resp = await _post(
client,
f"{settings.ledger_url}/internal/transactions",
{**payload, "request_id": request_id},
)
if create_resp.status_code == 404:
raise UpstreamFailure(404, "account not found")
if create_resp.status_code >= 500:
raise UpstreamFailure(create_resp.status_code, "ledger failed unexpectedly")
if create_resp.status_code >= 300:
raise UpstreamFailure(502, "ledger rejected the transaction")
created = create_resp.json()
tx_id = created["transaction_id"]
score_payload = {
"account_id": created["account_id"],
"holder_name": created["holder_name"],
"category_id": created["category_id"],
"request_id": request_id,
}
if no_batching:
item_count = chaos.synthetic_line_item_count()
scores = []
for _ in range(item_count):
item_payload = {**score_payload, "amount_minor": max(1, created["amount_minor"] // item_count)}
scores.append(await _score_with_retry(client, item_payload, retry_storm))
risk_score = max(s["risk_score"] for s in scores)
else:
score_payload["amount_minor"] = created["amount_minor"]
scored = await _score_with_retry(client, score_payload, retry_storm)
risk_score = scored["risk_score"]
decision = "rejected" if risk_score > REJECT_THRESHOLD else "settled"
action = "reject" if decision == "rejected" else "settle"
settle_resp = await _post(
client,
f"{settings.ledger_url}/internal/transactions/{tx_id}/{action}",
{"risk_score": risk_score, "request_id": request_id},
)
if settle_resp.status_code >= 300:
raise UpstreamFailure(502, "ledger failed to finalise the transaction")
settled = settle_resp.json()
try:
await redis_client.xadd(
settings.stream_name,
{
"transaction_id": tx_id,
"account_id": created["account_id"],
"kind": created["kind"],
"amount_minor": str(created["amount_minor"]),
"status": settled["status"],
"risk_score": str(risk_score),
},
)
except Exception:
logger.exception(f"failed to publish transaction {tx_id} to the stream")
summary = {}
try:
summary_resp = await client.get(
f"{settings.insights_url}/internal/accounts/{created['account_id']}/quick_summary"
)
if summary_resp.status_code == 200:
summary = summary_resp.json()
except httpx.RequestError:
logger.exception(f"insights summary unavailable for request_id={request_id}")
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(f"finished transaction {tx_id} status={settled['status']} request_id={request_id}")
return {
"transaction_id": tx_id,
"status": settled["status"],
"risk_score": risk_score,
"balance_minor": settled["balance_minor"],
"summary": summary,
"request_id": request_id,
"elapsed_ms": elapsed_ms,
}
+149
View File
@@ -0,0 +1,149 @@
const accountSelect = document.getElementById("account-select");
const txForm = document.getElementById("tx-form");
const resultJson = document.getElementById("result-json");
const resultRtt = document.getElementById("result-rtt");
const resultRequestId = document.getElementById("result-request-id");
const burstButton = document.getElementById("burst-button");
const burstN = document.getElementById("burst-n");
const burstTableBody = document.querySelector("#burst-table tbody");
const chaosFlagsDiv = document.getElementById("chaos-flags");
const healthStrip = document.getElementById("health-strip");
async function loadAccounts() {
const resp = await fetch("/api/accounts");
const accounts = await resp.json();
accountSelect.innerHTML = accounts
.slice(0, 200)
.map((a) => `<option value="${a.id}">${a.holder_name} (${a.currency})</option>`)
.join("");
}
txForm.addEventListener("submit", async (e) => {
e.preventDefault();
const payload = {
account_id: accountSelect.value,
kind: document.querySelector('input[name="kind"]:checked').value,
amount_minor: Number(document.getElementById("amount-input").value),
currency: "EUR",
merchant_name: document.getElementById("merchant-input").value,
};
const start = performance.now();
const resp = await fetch("/api/transactions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
const rtt = performance.now() - start;
const body = await resp.json();
resultRtt.textContent = rtt.toFixed(1);
resultRequestId.textContent = body.request_id || "--";
resultJson.textContent = JSON.stringify(body, null, 2);
});
function percentile(sorted, p) {
if (sorted.length === 0) return 0;
const rank = Math.max(1, Math.min(sorted.length, Math.ceil(p * sorted.length)));
return sorted[rank - 1];
}
burstButton.addEventListener("click", async () => {
const n = Number(burstN.value);
const accounts = Array.from(accountSelect.options).map((o) => o.value);
burstTableBody.innerHTML = `<tr><td colspan="8">running ${n} requests...</td></tr>`;
const latencies = [];
let success = 0;
let errors = 0;
await Promise.all(
Array.from({ length: n }, async () => {
const accountId = accounts[Math.floor(Math.random() * accounts.length)];
const start = performance.now();
try {
const resp = await fetch("/api/transactions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
account_id: accountId,
kind: "purchase",
amount_minor: 500,
currency: "EUR",
merchant_name: "Load Burst",
}),
});
latencies.push(performance.now() - start);
if (resp.ok) success += 1;
else errors += 1;
} catch (e) {
latencies.push(performance.now() - start);
errors += 1;
}
})
);
latencies.sort((a, b) => a - b);
const mean = latencies.reduce((a, b) => a + b, 0) / latencies.length;
const row = [
latencies.length,
success,
errors,
latencies[0].toFixed(1),
mean.toFixed(1),
percentile(latencies, 0.5).toFixed(1),
percentile(latencies, 0.95).toFixed(1),
latencies[latencies.length - 1].toFixed(1),
];
burstTableBody.innerHTML = `<tr>${row.map((v) => `<td>${v}</td>`).join("")}</tr>`;
});
async function loadChaos() {
const resp = await fetch("/api/admin/chaos");
const { flags } = await resp.json();
chaosFlagsDiv.innerHTML = Object.entries(flags)
.map(
([flag, enabled]) => `
<label>
<input type="checkbox" data-flag="${flag}" ${enabled ? "checked" : ""}>
${flag}
</label>`
)
.join("");
chaosFlagsDiv.querySelectorAll("input[type=checkbox]").forEach((el) => {
el.addEventListener("change", async () => {
await fetch("/api/admin/chaos", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ flag: el.dataset.flag, enabled: el.checked }),
});
});
});
}
document.querySelectorAll("#chaos-profiles button").forEach((btn) => {
btn.addEventListener("click", async () => {
await fetch("/api/admin/chaos/profile", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ profile: btn.dataset.profile }),
});
await loadChaos();
});
});
async function pollHealth() {
try {
const resp = await fetch("/api/health/all");
const { services } = await resp.json();
healthStrip.innerHTML = Object.entries(services)
.map(([name, status]) => `<span class="dot ${status === "ok" ? "green" : "red"}">${name}</span>`)
.join("");
} catch (e) {
// ignore transient polling failures
}
}
loadAccounts();
loadChaos();
pollHealth();
setInterval(pollHealth, 5000);
+80
View File
@@ -0,0 +1,80 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Ledgerline</title>
<link rel="stylesheet" href="/static/style.css">
</head>
<body>
<header>
<h1>Ledgerline</h1>
<p class="tagline">a deliberately imperfect payments backend</p>
</header>
<main>
<section id="submit-panel">
<h2>Submit a transaction</h2>
<form id="tx-form">
<label>Account
<select id="account-select"></select>
</label>
<label>Amount (minor units)
<input type="number" id="amount-input" value="1000" min="0">
</label>
<label>Merchant
<input type="text" id="merchant-input" value="Corner Store">
</label>
<fieldset>
<legend>Kind</legend>
<label><input type="radio" name="kind" value="deposit" checked> deposit</label>
<label><input type="radio" name="kind" value="withdrawal"> withdrawal</label>
<label><input type="radio" name="kind" value="purchase"> purchase</label>
</fieldset>
<button type="submit">Submit</button>
</form>
</section>
<section id="result-panel">
<h2>Result</h2>
<p>Round-trip: <span id="result-rtt">--</span> ms &middot; request_id: <code id="result-request-id">--</code></p>
<pre id="result-json">(nothing yet)</pre>
</section>
<section id="load-panel">
<h2>Load burst</h2>
<label>N
<select id="burst-n">
<option value="10">10</option>
<option value="50">50</option>
<option value="200">200</option>
</select>
</label>
<button id="burst-button">Fire burst</button>
<table id="burst-table">
<thead>
<tr><th>count</th><th>success</th><th>errors</th><th>min</th><th>mean</th><th>p50</th><th>p95</th><th>max</th></tr>
</thead>
<tbody><tr><td colspan="8">no burst run yet</td></tr></tbody>
</table>
</section>
<section id="chaos-panel">
<h2>Chaos</h2>
<div id="chaos-flags"></div>
<div id="chaos-profiles">
<button data-profile="clean">clean</button>
<button data-profile="light">light</button>
<button data-profile="nightmare">nightmare</button>
<button data-profile="random">random</button>
</div>
</section>
<section id="health-panel">
<h2>Service health</h2>
<div id="health-strip"></div>
</section>
</main>
<script src="/static/app.js"></script>
</body>
</html>
+184
View File
@@ -0,0 +1,184 @@
:root {
color-scheme: dark;
--bg: #0f1117;
--panel: #171a23;
--border: #2a2e3a;
--text: #d8dce6;
--muted: #7b8296;
--accent: #5fd0a8;
--danger: #e0685f;
--mono: "SF Mono", "Fira Code", Consolas, monospace;
}
* {
box-sizing: border-box;
}
body {
background: var(--bg);
color: var(--text);
font-family: var(--mono);
margin: 0;
padding: 0 1.5rem 3rem;
}
header {
padding: 2rem 0 1rem;
border-bottom: 1px solid var(--border);
}
header h1 {
margin: 0;
font-size: 1.6rem;
letter-spacing: 0.02em;
}
.tagline {
color: var(--muted);
margin: 0.25rem 0 0;
font-size: 0.85rem;
}
main {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
gap: 1.5rem;
margin-top: 1.5rem;
}
section {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 6px;
padding: 1rem 1.25rem;
}
section h2 {
margin-top: 0;
font-size: 1rem;
color: var(--accent);
text-transform: uppercase;
letter-spacing: 0.05em;
}
label {
display: block;
margin-bottom: 0.6rem;
font-size: 0.85rem;
color: var(--muted);
}
input, select, button {
font-family: var(--mono);
background: #0c0e13;
color: var(--text);
border: 1px solid var(--border);
border-radius: 4px;
padding: 0.4rem 0.5rem;
margin-top: 0.2rem;
width: 100%;
}
fieldset {
border: 1px solid var(--border);
border-radius: 4px;
margin-bottom: 0.6rem;
}
fieldset label {
display: inline-block;
width: auto;
margin-right: 1rem;
}
button {
cursor: pointer;
background: var(--accent);
color: #0c0e13;
font-weight: 600;
border: none;
margin-top: 0.5rem;
transition: opacity 0.15s ease;
}
button:hover {
opacity: 0.85;
}
#chaos-profiles {
display: flex;
gap: 0.5rem;
margin-top: 0.75rem;
}
#chaos-profiles button {
width: auto;
flex: 1;
}
pre#result-json {
background: #0c0e13;
border: 1px solid var(--border);
border-radius: 4px;
padding: 0.75rem;
overflow-x: auto;
font-size: 0.8rem;
max-height: 260px;
}
code {
background: #0c0e13;
padding: 0.1rem 0.3rem;
border-radius: 3px;
user-select: all;
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 0.75rem;
font-size: 0.8rem;
}
th, td {
border: 1px solid var(--border);
padding: 0.35rem 0.5rem;
text-align: right;
}
th:first-child, td:first-child {
text-align: left;
}
#chaos-flags label {
display: flex;
align-items: center;
gap: 0.4rem;
}
#chaos-flags input {
width: auto;
}
#health-strip {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
}
.dot {
border-radius: 999px;
padding: 0.25rem 0.6rem;
font-size: 0.75rem;
border: 1px solid var(--border);
}
.dot.green {
color: var(--accent);
border-color: var(--accent);
}
.dot.red {
color: var(--danger);
border-color: var(--danger);
}
View File
+82
View File
@@ -0,0 +1,82 @@
import sqlalchemy as sa
# Deliberately the same shape of query the test suite's own EXPLAIN check runs
# against the schema directly (tests/test_pathologies.py, 6.4) -- kept here
# only for the honest (non-n_plus_one) path; that test doesn't call this code.
_N_PLUS_ONE_ROW_CAP = 1500
async def account_exists(session, account_id: str) -> bool:
result = await session.execute(sa.text("SELECT 1 FROM accounts WHERE id = :id"), {"id": account_id})
return result.first() is not None
async def compute_summary(session, account_id: str, *, n_plus_one: bool) -> dict:
if n_plus_one:
rows = await _n_plus_one_summary(session, account_id)
else:
rows = await _joined_summary(session, account_id)
total_count = 0
total_amount = 0
by_category: dict = {}
for r in rows:
total_count += 1
total_amount += r["amount_minor"]
label = r["label"] or "uncategorized"
entry = by_category.setdefault(label, {"count": 0, "total_amount_minor": 0})
entry["count"] += 1
entry["total_amount_minor"] += r["amount_minor"]
return {
"account_id": account_id,
"window_days": 120,
"transaction_count": total_count,
"total_amount_minor": total_amount,
"by_category": by_category,
}
async def _joined_summary(session, account_id: str) -> list[dict]:
"""The honest path: a single JOIN, one statement."""
result = await session.execute(
sa.text(
"SELECT t.amount_minor, mc.label FROM transactions t "
"LEFT JOIN merchant_categories mc ON mc.id = t.category_id "
"WHERE t.account_id = :aid AND t.created_at >= now() - interval '120 days'"
),
{"aid": account_id},
)
return [dict(row) for row in result.mappings().all()]
async def _n_plus_one_summary(session, account_id: str) -> list[dict]:
"""n_plus_one: instead of the JOIN above, fetch rows without one and issue
a separate category lookup per row -- phase-0-base-project-spec.md §7.
Capped so the pathology stays a genuine multiplication of statements
(see tests/test_pathologies.py 6.3) without turning a single request into
an unbounded number of round trips on a high-volume hot account."""
result = await session.execute(
sa.text(
"SELECT amount_minor, category_id FROM transactions "
"WHERE account_id = :aid AND created_at >= now() - interval '120 days' "
f"LIMIT {_N_PLUS_ONE_ROW_CAP}"
),
{"aid": account_id},
)
tx_rows = result.mappings().all()
rows = []
for tx_row in tx_rows:
label = None
if tx_row["category_id"] is not None:
cat_result = await session.execute(
sa.text("SELECT label FROM merchant_categories WHERE id = :id"),
{"id": tx_row["category_id"]},
)
cat_row = cat_result.first()
if cat_row is not None:
label = cat_row.label
rows.append({"amount_minor": tx_row["amount_minor"], "label": label})
return rows
+120
View File
@@ -0,0 +1,120 @@
import asyncio
import json
import time
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from common.chaos_flags import flags_fingerprint
from common.logging_config import configure_logging
from common.redis_client import redis_client
from common.settings import settings
import sqlalchemy as sa
from . import chaos
from .analytics import account_exists, compute_summary
from .db import NormalSession, engine, session_factory
SERVICE_NAME = "insights"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
# Cache-stampede protection for the Redis-cached branch below: a burst of
# concurrent requests for the same not-yet-cached (account, flags) key would
# otherwise all pay the full computation cost at once instead of one paying
# it and the rest waiting on the result. Not needed for unbounded_cache,
# which never caches by design.
_inflight_locks: dict[str, asyncio.Lock] = {}
def _lock_for(key: str) -> asyncio.Lock:
lock = _inflight_locks.get(key)
if lock is None:
lock = asyncio.Lock()
_inflight_locks[key] = lock
return lock
@app.get("/")
async def root():
return {"service": SERVICE_NAME, "version": "0.1.0"}
@app.get("/healthz")
async def healthz(request: Request):
if request.headers.get("X-Ledgerline-Probe") == "chatty-health":
logger.info("received chatty-health healthz probe from gateway")
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
try:
async with engine.connect() as conn:
await conn.exec_driver_sql("SELECT 1")
except Exception as exc:
logger.exception(f"readyz check failed: {exc}")
return JSONResponse(status_code=503, content={"status": "unavailable"})
return {"status": "ok"}
@app.get("/internal/accounts/{account_id}/quick_summary")
async def quick_summary(account_id: str):
"""A cheap, account-primary-key-only lookup embedded in every transaction
response (phase-0-base-project-spec.md §6.1's "summary" field) -- deliberately
separate from the 120-day aggregation below (§6.2), which is the one where
n_plus_one and the missing index are meant to bite. That aggregation scans
`transactions` by account_id with no supporting index on purpose; embedding
it in every single transaction's response would make the missing-index
pathology fire on every request instead of only on the analytics endpoint."""
async with NormalSession() as session:
result = await session.execute(
sa.text("SELECT currency, balance_minor FROM accounts WHERE id = :id"),
{"id": account_id},
)
row = result.first()
if row is None:
return JSONResponse(status_code=404, content={"error": "account not found"})
return {"account_id": account_id, "currency": row.currency, "balance_minor": row.balance_minor}
@app.get("/internal/accounts/{account_id}/summary")
async def summary(account_id: str):
n_plus_one = await chaos.n_plus_one_enabled()
cold_cache = await chaos.cold_cache_enabled()
unbounded_cache = await chaos.unbounded_cache_enabled()
fingerprint = await flags_fingerprint()
# unbounded_cache means every single request is a guaranteed fresh miss
# needing a real connection -- under concurrent load that alone is enough
# demand to exhaust a small pool, without needing n_plus_one too (see
# services/insights/db.py).
starved = unbounded_cache
async with session_factory(starved) as session:
if not await account_exists(session, account_id):
return JSONResponse(status_code=404, content={"error": "account not found"})
if unbounded_cache:
# Broken on purpose: keyed by a timestamp that's unique per call,
# so this never hits -- it only ever grows. See chaos.py.
key = (account_id, fingerprint, time.time_ns())
result = await compute_summary(session, account_id, n_plus_one=n_plus_one)
chaos.UNBOUNDED_CACHE[key] = result
return result
ttl = settings.summary_cache_ttl_seconds_cold if cold_cache else settings.summary_cache_ttl_seconds
cache_key = f"summary:{account_id}:{fingerprint}"
cached = await redis_client.get(cache_key)
if cached is not None:
return json.loads(cached)
async with _lock_for(cache_key):
cached = await redis_client.get(cache_key)
if cached is not None:
return json.loads(cached)
async with session_factory(starved) as session:
result = await compute_summary(session, account_id, n_plus_one=n_plus_one)
await redis_client.setex(cache_key, ttl, json.dumps(result))
return result
+22
View File
@@ -0,0 +1,22 @@
"""n_plus_one, cold_cache, unbounded_cache -- phase-0-base-project-spec.md §7.
All three are owned by `insights`."""
from common.chaos_flags import is_enabled
async def n_plus_one_enabled() -> bool:
return await is_enabled("n_plus_one")
async def cold_cache_enabled() -> bool:
return await is_enabled("cold_cache")
async def unbounded_cache_enabled() -> bool:
return await is_enabled("unbounded_cache")
# unbounded_cache: memoize summaries in a module-level dict keyed by
# (account_id, timestamp_ms) -- effectively never hits, never evicts, grows
# forever. This is the pathology itself, not a fix for anything -- never
# clear it, never cap it.
UNBOUNDED_CACHE: dict = {}
+25
View File
@@ -0,0 +1,25 @@
"""insights gets its own engines, separate from common.db's shared default,
so pool sizing here can be tuned independently of sanctions/notifier/audit
(which also import common.db).
Two pools, selected per-request: a generous default, and a small one used
whenever unbounded_cache is on -- every request is then a guaranteed fresh
miss needing a real connection, and under concurrent load that alone is
enough demand to exhaust a small pool. This is a realistic capacity limit
that emerges under load, not a 14th named chaos flag."""
from sqlalchemy.ext.asyncio import async_sessionmaker
from common.db import make_engine
normal_engine = make_engine(pool_size=20, max_overflow=10)
starved_engine = make_engine(pool_size=5, max_overflow=3, pool_timeout=0.5)
NormalSession = async_sessionmaker(normal_engine, expire_on_commit=False)
StarvedSession = async_sessionmaker(starved_engine, expire_on_commit=False)
# Exposed for readyz's connectivity check.
engine = normal_engine
def session_factory(starved: bool):
return StarvedSession() if starved else NormalSession()
View File
+146
View File
@@ -0,0 +1,146 @@
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, field_validator
from common.logging_config import configure_logging
from . import chaos
from .db import normal_engine, session_factory
from . import repository
SERVICE_NAME = "ledger"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
class CreateTransactionRequest(BaseModel):
account_id: str
kind: str
amount_minor: int
currency: str
merchant_name: str
category_id: int | None = None
request_id: str = "unknown"
@field_validator("amount_minor")
@classmethod
def deposit_amount_must_be_non_negative(cls, v, info):
kind = info.data.get("kind")
if kind == "deposit" and v < 0:
raise ValueError("amount_minor must be non-negative for a deposit")
return v
class SettleRequest(BaseModel):
risk_score: float
request_id: str = "unknown"
@app.exception_handler(Exception)
async def unhandled_exception_handler(request: Request, exc: Exception):
logger.exception(f"unhandled exception on {request.url.path}: {exc}")
return JSONResponse(status_code=500, content={"error": "internal error"})
@app.get("/")
async def root():
return {"service": SERVICE_NAME, "version": "0.1.0"}
@app.get("/healthz")
async def healthz(request: Request):
if request.headers.get("X-Ledgerline-Probe") == "chatty-health":
logger.info("received chatty-health healthz probe from gateway")
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
try:
async with normal_engine.connect() as conn:
await conn.exec_driver_sql("SELECT 1")
except Exception as exc:
logger.exception(f"readyz check failed: {exc}")
return JSONResponse(status_code=503, content={"status": "unavailable"})
return {"status": "ok"}
@app.get("/internal/accounts")
async def list_accounts():
async with session_factory(starved=False) as session:
rows = await repository.list_accounts(session)
return [dict(row) for row in rows]
@app.post("/internal/transactions")
async def create_transaction(body: CreateTransactionRequest):
logger.info(f"processing tx for account {body.account_id}, amount {body.amount_minor} request_id={body.request_id}")
starved = await chaos.pool_starvation_enabled()
poison_enabled = await chaos.poison_data_enabled()
poison_tx_id = await chaos.find_poison_transaction_id(body.account_id)
async with session_factory(starved=starved) as session:
account = await repository.get_account(session, body.account_id)
if account is None:
return JSONResponse(status_code=404, content={"error": "account not found"})
if poison_tx_id is not None and poison_enabled:
poison_row = await repository.get_transaction_currency(session, poison_tx_id)
# poison_data: skip input validation on the seeded malformed rows
# (§8) -- this naive access raises on the NULL currency on purpose.
_ = poison_row.currency.upper()
tx_id = await repository.create_transaction(
session,
account_id=body.account_id,
kind=body.kind,
amount_minor=body.amount_minor,
currency=body.currency,
merchant_name=body.merchant_name,
category_id=body.category_id,
)
return JSONResponse(
status_code=201,
content={
"transaction_id": tx_id,
"account_id": body.account_id,
"holder_name": account.holder_name,
"kind": body.kind,
"amount_minor": body.amount_minor,
"category_id": body.category_id,
"status": "pending",
},
)
@app.post("/internal/transactions/{tx_id}/settle")
async def settle(tx_id: str, body: SettleRequest):
starved = await chaos.pool_starvation_enabled()
row_lock = await chaos.row_lock_contention_enabled()
async with session_factory(starved=starved) as session:
tx = await repository.get_transaction(session, tx_id)
if tx is None:
return JSONResponse(status_code=404, content={"error": "transaction not found"})
account = await repository.get_account(session, tx.account_id)
use_lock = row_lock and bool(account.is_hot)
balance = await repository.settle_transaction(session, tx=tx, risk_score=body.risk_score, use_row_lock=use_lock)
logger.info(f"settled transaction {tx_id} request_id={body.request_id}")
return {"status": "settled", "balance_minor": balance}
@app.post("/internal/transactions/{tx_id}/reject")
async def reject(tx_id: str, body: SettleRequest):
starved = await chaos.pool_starvation_enabled()
async with session_factory(starved=starved) as session:
tx = await repository.get_transaction(session, tx_id)
if tx is None:
return JSONResponse(status_code=404, content={"error": "transaction not found"})
balance = await repository.reject_transaction(session, tx=tx, risk_score=body.risk_score)
logger.info(f"rejected transaction {tx_id} request_id={body.request_id}")
return {"status": "rejected", "balance_minor": balance}
+28
View File
@@ -0,0 +1,28 @@
"""pool_starvation, row_lock_contention, poison_data -- phase-0-base-project-spec.md
§7. All three are owned by `ledger`.
"""
from common.chaos_flags import is_enabled
from common.redis_client import redis_client
# Populated once by scripts/seed.py: account_id -> id of that account's one
# deliberately NULL-currency transaction (§8). A Redis hash lookup, not a
# `transactions` scan by account_id -- that column has no supporting index
# on purpose (§5), and the poison_data pathology shouldn't have to pay for a
# full-table seq scan on every single request just to find its own trigger.
POISON_TX_KEY = "poison:tx_by_account"
async def pool_starvation_enabled() -> bool:
return await is_enabled("pool_starvation")
async def row_lock_contention_enabled() -> bool:
return await is_enabled("row_lock_contention")
async def poison_data_enabled() -> bool:
return await is_enabled("poison_data")
async def find_poison_transaction_id(account_id: str) -> str | None:
return await redis_client.hget(POISON_TX_KEY, account_id)
+17
View File
@@ -0,0 +1,17 @@
"""Two pre-built engines so pool_starvation (phase-0-base-project-spec.md §7)
can be toggled live, with no restart: the flag picks which already-initialised
pool a request borrows a connection from, rather than resizing a pool in
place (SQLAlchemy pools aren't resizable live)."""
from sqlalchemy.ext.asyncio import async_sessionmaker
from common.db import make_engine
normal_engine = make_engine(pool_size=20, max_overflow=10)
starved_engine = make_engine(pool_size=2, max_overflow=0, pool_timeout=10)
NormalSession = async_sessionmaker(normal_engine, expire_on_commit=False)
StarvedSession = async_sessionmaker(starved_engine, expire_on_commit=False)
def session_factory(starved: bool):
return StarvedSession() if starved else NormalSession()
+122
View File
@@ -0,0 +1,122 @@
import asyncio
import uuid
import sqlalchemy as sa
KIND_SIGN = {"deposit": 1, "withdrawal": -1, "purchase": -1}
ROW_LOCK_HOLD_SECONDS = 0.15
async def get_account(session, account_id: str):
result = await session.execute(
sa.text("SELECT id, holder_name, currency, balance_minor, is_hot FROM accounts WHERE id = :id"),
{"id": account_id},
)
return result.first()
async def list_accounts(session, limit: int = 1000):
result = await session.execute(
sa.text(
"SELECT id, holder_name, currency, balance_minor FROM accounts ORDER BY created_at LIMIT :limit"
),
{"limit": limit},
)
return result.mappings().all()
async def get_transaction_currency(session, tx_id: str):
"""Backs the poison_data pathology: fetches one known-malformed transaction
(NULL currency, §8) by primary key -- a cheap, always-fast lookup regardless
of the deliberately-missing account_id index, since which transaction to
fetch is looked up by account_id in Redis first (chaos.py's
find_poison_transaction_id), not by scanning `transactions` here."""
result = await session.execute(
sa.text("SELECT currency FROM transactions WHERE id = :id"),
{"id": tx_id},
)
return result.first()
async def create_transaction(
session, *, account_id, kind, amount_minor, currency, merchant_name, category_id
):
tx_id = str(uuid.uuid4())
await session.execute(
sa.text(
"INSERT INTO transactions "
"(id, account_id, kind, amount_minor, currency, merchant_name, category_id, status) "
"VALUES (:id, :account_id, :kind, :amount_minor, :currency, :merchant_name, :category_id, 'pending')"
),
{
"id": tx_id,
"account_id": account_id,
"kind": kind,
"amount_minor": amount_minor,
"currency": currency,
"merchant_name": merchant_name,
"category_id": category_id,
},
)
await session.commit()
return tx_id
async def get_transaction(session, tx_id: str):
result = await session.execute(
sa.text(
"SELECT id, account_id, kind, amount_minor, currency, merchant_name, "
"category_id, status FROM transactions WHERE id = :id"
),
{"id": tx_id},
)
return result.first()
async def settle_transaction(session, *, tx, risk_score, use_row_lock: bool):
"""Applies the balance delta and marks the transaction settled. When
use_row_lock is true (row_lock_contention on a hot account), takes
SELECT ... FOR UPDATE and holds the transaction open for an extra 150ms
before committing -- phase-0-base-project-spec.md §7."""
delta = KIND_SIGN[tx.kind] * tx.amount_minor
if use_row_lock:
result = await session.execute(
sa.text("SELECT balance_minor FROM accounts WHERE id = :id FOR UPDATE"),
{"id": tx.account_id},
)
result.first()
await session.execute(
sa.text("UPDATE accounts SET balance_minor = balance_minor + :delta WHERE id = :id"),
{"delta": delta, "id": tx.account_id},
)
await asyncio.sleep(ROW_LOCK_HOLD_SECONDS)
else:
await session.execute(
sa.text("UPDATE accounts SET balance_minor = balance_minor + :delta WHERE id = :id"),
{"delta": delta, "id": tx.account_id},
)
await session.execute(
sa.text("UPDATE transactions SET status = 'settled', risk_score = :risk_score WHERE id = :id"),
{"risk_score": risk_score, "id": tx.id},
)
await session.commit()
result = await session.execute(
sa.text("SELECT balance_minor FROM accounts WHERE id = :id"), {"id": tx.account_id}
)
return result.scalar_one()
async def reject_transaction(session, *, tx, risk_score):
await session.execute(
sa.text("UPDATE transactions SET status = 'rejected', risk_score = :risk_score WHERE id = :id"),
{"risk_score": risk_score, "id": tx.id},
)
await session.commit()
result = await session.execute(
sa.text("SELECT balance_minor FROM accounts WHERE id = :id"), {"id": tx.account_id}
)
return result.scalar_one()
View File
+49
View File
@@ -0,0 +1,49 @@
from contextlib import asynccontextmanager
import asyncio
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from common.db import engine
from common.logging_config import configure_logging
from common.redis_client import redis_client
from . import consumer
SERVICE_NAME = "notifier"
logger = configure_logging(SERVICE_NAME)
@asynccontextmanager
async def lifespan(app: FastAPI):
task = asyncio.create_task(consumer.run())
yield
task.cancel()
app = FastAPI(title=SERVICE_NAME, lifespan=lifespan)
@app.get("/")
async def root():
return {"service": SERVICE_NAME, "version": "0.1.0"}
@app.get("/healthz")
async def healthz(request: Request):
if request.headers.get("X-Ledgerline-Probe") == "chatty-health":
logger.info("received chatty-health healthz probe from gateway")
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
try:
async with engine.connect() as conn:
await conn.exec_driver_sql("SELECT 1")
await redis_client.ping()
except Exception as exc:
logger.exception(f"readyz check failed: {exc}")
return JSONResponse(status_code=503, content={"status": "unavailable"})
return {"status": "ok"}
+8
View File
@@ -0,0 +1,8 @@
"""slow_consumer -- phase-0-base-project-spec.md §7. Owned by `notifier`."""
from common.chaos_flags import is_enabled
SLEEP_SECONDS = 0.25
async def slow_consumer_enabled() -> bool:
return await is_enabled("slow_consumer")
+78
View File
@@ -0,0 +1,78 @@
import asyncio
import logging
import random
import uuid
import sqlalchemy as sa
from common.db import Session
from common.redis_client import redis_client
from common.settings import settings
from . import chaos
logger = logging.getLogger("notifier")
GROUP = "notifier"
CONSUMER = "notifier-1"
async def ensure_group():
try:
await redis_client.xgroup_create(settings.stream_name, GROUP, id="$", mkstream=True)
except Exception as exc:
if "BUSYGROUP" not in str(exc):
raise
async def _handle(fields: dict):
if await chaos.slow_consumer_enabled():
await asyncio.sleep(chaos.SLEEP_SECONDS)
async with Session() as session:
await session.execute(
sa.text(
"INSERT INTO notifications (id, transaction_id, channel, status, attempts, sent_at) "
"VALUES (:id, :tx_id, :channel, 'sent', 1, now())"
),
{
"id": str(uuid.uuid4()),
"tx_id": fields.get("transaction_id"),
"channel": random.choice(["email", "push", "sms"]),
},
)
await session.commit()
async def run():
"""Background consumer loop for the `notifier` consumer group -- a second,
independent group (`audit`) reads the same stream in services/audit/worker.py.
Redis Streams carry no automatic context propagation, deliberately."""
await ensure_group()
logger.info("notifier consumer started")
while True:
try:
resp = await redis_client.xreadgroup(
groupname=GROUP,
consumername=CONSUMER,
streams={settings.stream_name: ">"},
count=10,
block=2000,
)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("notifier consumer read failed")
await asyncio.sleep(1)
continue
if not resp:
continue
for _, messages in resp:
for msg_id, fields in messages:
try:
await _handle(fields)
except Exception:
logger.exception(f"notifier failed to process message {msg_id}")
await redis_client.xack(settings.stream_name, GROUP, msg_id)
View File
+63
View File
@@ -0,0 +1,63 @@
import asyncio
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from common.db import engine
from common.logging_config import configure_logging
from common.redis_client import redis_client
from . import chaos
from .repository import is_match
SERVICE_NAME = "sanctions"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
class LookupRequest(BaseModel):
holder_name: str
request_id: str = "unknown"
@app.get("/")
async def root():
return {"service": SERVICE_NAME, "version": "0.1.0"}
@app.get("/healthz")
async def healthz(request: Request):
if request.headers.get("X-Ledgerline-Probe") == "chatty-health":
logger.info(f"received chatty-health healthz probe from gateway")
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
try:
async with engine.connect() as conn:
await conn.exec_driver_sql("SELECT 1")
except Exception as exc:
logger.exception(f"readyz check failed: {exc}")
return JSONResponse(status_code=503, content={"status": "unavailable"})
return {"status": "ok"}
@app.post("/internal/lookup")
async def lookup(body: LookupRequest):
logger.info(f"lookup for holder_name={body.holder_name!r} request_id={body.request_id}")
if await chaos.sanctions_errors_enabled() and chaos.should_error():
logger.info(f"sanctions_errors chaos triggered for request_id={body.request_id}")
return JSONResponse(status_code=503, content={"error": "sanctions vendor unavailable"})
if await chaos.slow_sanctions_enabled():
delay = chaos.sample_delay_seconds()
logger.info(f"slow_sanctions chaos sleeping {delay:.3f}s for request_id={body.request_id}")
await asyncio.sleep(delay)
match = await is_match(body.holder_name)
if match:
return {"matched": True, **match}
return {"matched": False, "country": None, "list_source": None}
+37
View File
@@ -0,0 +1,37 @@
"""slow_sanctions and sanctions_errors -- phase-0-base-project-spec.md §7.
Both flags belong to `sanctions`, the fake third-party watchlist vendor.
"""
import math
import random
from common.chaos_flags import is_enabled
_MEDIAN_SECONDS = 0.04
_P99_SECONDS = 1.8
_MU = math.log(_MEDIAN_SECONDS)
_Z99 = 2.326347874
_SIGMA = (math.log(_P99_SECONDS) - _MU) / _Z99
_TIMEOUT_TRIGGER_PROBABILITY = 0.015
_TIMEOUT_TRIGGER_SECONDS = 5.0
_ERROR_PROBABILITY = 0.03
async def slow_sanctions_enabled() -> bool:
return await is_enabled("slow_sanctions")
async def sanctions_errors_enabled() -> bool:
return await is_enabled("sanctions_errors")
def sample_delay_seconds() -> float:
"""Log-normal delay: median ~40ms, p99 ~1.8s. 1.5% of calls sleep 5s,
which exceeds the caller's 3s downstream timeout and surfaces as one."""
if random.random() < _TIMEOUT_TRIGGER_PROBABILITY:
return _TIMEOUT_TRIGGER_SECONDS
return random.lognormvariate(_MU, _SIGMA)
def should_error() -> bool:
return random.random() < _ERROR_PROBABILITY
+19
View File
@@ -0,0 +1,19 @@
import sqlalchemy as sa
from common.db import Session
async def is_match(holder_name: str) -> dict | None:
"""Case-insensitive exact match against sanctions_entries.normalized_name."""
async with Session() as session:
result = await session.execute(
sa.text(
"SELECT country, list_source FROM sanctions_entries "
"WHERE lower(normalized_name) = lower(:name) LIMIT 1"
),
{"name": holder_name},
)
row = result.first()
if row is None:
return None
return {"country": row.country, "list_source": row.list_source}