Implement Phase 0b base project against the frozen test contract

All seven services (gateway, ledger, fraud, sanctions, insights, notifier,
audit), db/schema.sql plus optional_indexes.sql, scripts/seed.py and
smoke.py, the gateway-served frontend, and docker-compose.yml. Plain-text
stdlib logging and a weak manual X-Request-Id only -- no observability yet,
by design, per test_no_observability.py. Implements all 13 named chaos-flag
pathologies from docs/PATHOLOGIES.md behind Redis-backed runtime flags.

make smoke and make test-contract pass; make test-baseline and
make test-pathology pass except for a handful of load-sensitive tests
documented in tests/README.md rather than papered over.
This commit is contained in:
Movses Aslanyan
2026-09-05 08:12:30 +03:30
parent ccf1783779
commit 91afea92a1
51 changed files with 2757 additions and 0 deletions
View File
+151
View File
@@ -0,0 +1,151 @@
import asyncio
import uuid
from pathlib import Path
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, field_validator
from common.chaos_flags import ALL_FLAGS, get_flags, set_flag, set_profile
from common.logging_config import configure_logging
from common.settings import settings
from . import orchestrator
from .orchestrator import UpstreamFailure
SERVICE_NAME = "gateway"
logger = configure_logging(SERVICE_NAME)
app = FastAPI(title=SERVICE_NAME)
STATIC_DIR = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
# A shared, connection-pooling client for the simple read-only proxy routes
# below -- opening a fresh httpx.AsyncClient (and TCP connection) per request
# is real, needless overhead under concurrent load. orchestrator.py's
# downstream_client() stays per-request since it carries per-transaction
# headers (X-Request-Id).
_proxy_client = httpx.AsyncClient(
timeout=settings.downstream_timeout_seconds,
limits=httpx.Limits(max_connections=100, max_keepalive_connections=50),
)
DOWNSTREAM_URLS = {
"ledger": settings.ledger_url,
"fraud": settings.fraud_url,
"sanctions": settings.sanctions_url,
"insights": settings.insights_url,
"notifier": settings.notifier_url,
}
class TransactionRequest(BaseModel):
account_id: str
kind: str
amount_minor: int
currency: str
merchant_name: str
category_id: int | None = None
@field_validator("kind")
@classmethod
def kind_must_be_known(cls, v):
if v not in ("deposit", "withdrawal", "purchase"):
raise ValueError("kind must be one of deposit, withdrawal, purchase")
return v
@field_validator("amount_minor")
@classmethod
def deposit_amount_must_be_non_negative(cls, v, info):
if info.data.get("kind") == "deposit" and v < 0:
raise ValueError("amount_minor must be non-negative for a deposit")
return v
class ChaosFlagRequest(BaseModel):
flag: str
enabled: bool
class ChaosProfileRequest(BaseModel):
profile: str
@app.get("/", include_in_schema=False)
async def index():
"""Serves the frontend (phase-0-base-project-spec.md §10). This overrides
§4's generic `GET / -> {"service", "version"}` info route, which the
other five HTTP services still expose as-is."""
return FileResponse(STATIC_DIR / "index.html")
@app.get("/healthz")
async def healthz():
return {"status": "ok", "service": SERVICE_NAME}
@app.get("/readyz")
async def readyz():
return {"status": "ok"}
@app.get("/api/accounts")
async def list_accounts():
resp = await _proxy_client.get(f"{settings.ledger_url}/internal/accounts")
return JSONResponse(status_code=resp.status_code, content=resp.json())
@app.get("/api/accounts/{account_id}/insights")
async def account_insights(account_id: str):
resp = await _proxy_client.get(f"{settings.insights_url}/internal/accounts/{account_id}/summary")
return JSONResponse(status_code=resp.status_code, content=resp.json())
@app.post("/api/transactions", status_code=201)
async def submit_transaction(body: TransactionRequest):
request_id = str(uuid.uuid4())
try:
result = await orchestrator.create_transaction(body.model_dump(), request_id)
except UpstreamFailure as exc:
return JSONResponse(
status_code=exc.status_code, content={"error": exc.message, "request_id": request_id}
)
return JSONResponse(status_code=201, content=result)
@app.get("/api/health/all")
async def health_all():
async def check(name, url):
try:
resp = await _proxy_client.get(f"{url}/healthz", timeout=2.0)
return name, "ok" if resp.status_code == 200 else "down"
except httpx.RequestError:
return name, "down"
results = await asyncio.gather(*[check(name, url) for name, url in DOWNSTREAM_URLS.items()])
services = {"gateway": "ok", **dict(results)}
return {"services": services}
@app.get("/api/admin/chaos")
async def get_chaos():
return {"flags": await get_flags()}
@app.post("/api/admin/chaos")
async def post_chaos(body: ChaosFlagRequest):
if body.flag not in ALL_FLAGS:
return JSONResponse(status_code=400, content={"error": f"unknown flag {body.flag!r}"})
await set_flag(body.flag, body.enabled)
return {"flags": await get_flags()}
@app.post("/api/admin/chaos/profile")
async def post_chaos_profile(body: ChaosProfileRequest):
try:
flags = await set_profile(body.profile)
except ValueError as exc:
return JSONResponse(status_code=400, content={"error": str(exc)})
return {"flags": flags}
+23
View File
@@ -0,0 +1,23 @@
"""retry_storm, no_batching, chatty_health -- phase-0-base-project-spec.md §7.
All three are owned by `gateway`."""
import random
from common.chaos_flags import is_enabled
DOWNSTREAMS = ("ledger", "fraud", "sanctions", "insights", "notifier")
async def retry_storm_enabled() -> bool:
return await is_enabled("retry_storm")
async def no_batching_enabled() -> bool:
return await is_enabled("no_batching")
async def chatty_health_enabled() -> bool:
return await is_enabled("chatty_health")
def synthetic_line_item_count() -> int:
return random.randint(1, 5)
+167
View File
@@ -0,0 +1,167 @@
import logging
import time
import uuid
import httpx
from common.http import downstream_client
from common.redis_client import redis_client
from common.settings import settings
from . import chaos
logger = logging.getLogger("gateway")
REJECT_THRESHOLD = 0.8
class UpstreamFailure(Exception):
def __init__(self, status_code: int, message: str):
self.status_code = status_code
self.message = message
async def _post(client: httpx.AsyncClient, url: str, json: dict) -> httpx.Response:
try:
return await client.post(url, json=json)
except httpx.TimeoutException as exc:
raise UpstreamFailure(504, f"{url} timed out") from exc
except httpx.RequestError as exc:
raise UpstreamFailure(502, f"{url} unreachable") from exc
async def _score_once(client: httpx.AsyncClient, payload: dict) -> dict:
resp = await _post(client, f"{settings.fraud_url}/internal/score", payload)
if resp.status_code >= 300:
status = resp.status_code if resp.status_code in (502, 504) else 502
raise UpstreamFailure(status, "fraud scoring failed")
return resp.json()
async def _score_with_retry(client: httpx.AsyncClient, payload: dict, retry_storm: bool) -> dict:
"""retry_storm: on any non-2xx from fraud, retry 3 times immediately --
no backoff, no jitter, no circuit breaker (phase-0-base-project-spec.md §7).
"No circuit breaker" is taken literally here: once triggered, it fires all
3 retries unconditionally rather than stopping early on the first success
-- it never checks whether it still needs to. The last attempt's outcome
wins."""
attempts = 4 if retry_storm else 1
result = None
last_exc = None
for _ in range(attempts):
try:
result = await _score_once(client, payload)
last_exc = None
except UpstreamFailure as exc:
last_exc = exc
if last_exc is not None:
raise last_exc
return result
async def _run_chatty_health_probe(client: httpx.AsyncClient) -> None:
"""chatty_health: GET /healthz on all five downstreams, sequentially,
before doing any real work."""
headers = {"X-Ledgerline-Probe": "chatty-health"}
for name in chaos.DOWNSTREAMS:
url = getattr(settings, f"{name}_url")
try:
await client.get(f"{url}/healthz", headers=headers, timeout=2.0)
except httpx.RequestError:
pass
async def create_transaction(payload: dict, request_id: str) -> dict:
start = time.perf_counter()
logger.info(f"start transaction for account {payload['account_id']} request_id={request_id}")
retry_storm = await chaos.retry_storm_enabled()
no_batching = await chaos.no_batching_enabled()
chatty_health = await chaos.chatty_health_enabled()
async with downstream_client(request_id) as client:
if chatty_health:
await _run_chatty_health_probe(client)
create_resp = await _post(
client,
f"{settings.ledger_url}/internal/transactions",
{**payload, "request_id": request_id},
)
if create_resp.status_code == 404:
raise UpstreamFailure(404, "account not found")
if create_resp.status_code >= 500:
raise UpstreamFailure(create_resp.status_code, "ledger failed unexpectedly")
if create_resp.status_code >= 300:
raise UpstreamFailure(502, "ledger rejected the transaction")
created = create_resp.json()
tx_id = created["transaction_id"]
score_payload = {
"account_id": created["account_id"],
"holder_name": created["holder_name"],
"category_id": created["category_id"],
"request_id": request_id,
}
if no_batching:
item_count = chaos.synthetic_line_item_count()
scores = []
for _ in range(item_count):
item_payload = {**score_payload, "amount_minor": max(1, created["amount_minor"] // item_count)}
scores.append(await _score_with_retry(client, item_payload, retry_storm))
risk_score = max(s["risk_score"] for s in scores)
else:
score_payload["amount_minor"] = created["amount_minor"]
scored = await _score_with_retry(client, score_payload, retry_storm)
risk_score = scored["risk_score"]
decision = "rejected" if risk_score > REJECT_THRESHOLD else "settled"
action = "reject" if decision == "rejected" else "settle"
settle_resp = await _post(
client,
f"{settings.ledger_url}/internal/transactions/{tx_id}/{action}",
{"risk_score": risk_score, "request_id": request_id},
)
if settle_resp.status_code >= 300:
raise UpstreamFailure(502, "ledger failed to finalise the transaction")
settled = settle_resp.json()
try:
await redis_client.xadd(
settings.stream_name,
{
"transaction_id": tx_id,
"account_id": created["account_id"],
"kind": created["kind"],
"amount_minor": str(created["amount_minor"]),
"status": settled["status"],
"risk_score": str(risk_score),
},
)
except Exception:
logger.exception(f"failed to publish transaction {tx_id} to the stream")
summary = {}
try:
summary_resp = await client.get(
f"{settings.insights_url}/internal/accounts/{created['account_id']}/quick_summary"
)
if summary_resp.status_code == 200:
summary = summary_resp.json()
except httpx.RequestError:
logger.exception(f"insights summary unavailable for request_id={request_id}")
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(f"finished transaction {tx_id} status={settled['status']} request_id={request_id}")
return {
"transaction_id": tx_id,
"status": settled["status"],
"risk_score": risk_score,
"balance_minor": settled["balance_minor"],
"summary": summary,
"request_id": request_id,
"elapsed_ms": elapsed_ms,
}
+149
View File
@@ -0,0 +1,149 @@
const accountSelect = document.getElementById("account-select");
const txForm = document.getElementById("tx-form");
const resultJson = document.getElementById("result-json");
const resultRtt = document.getElementById("result-rtt");
const resultRequestId = document.getElementById("result-request-id");
const burstButton = document.getElementById("burst-button");
const burstN = document.getElementById("burst-n");
const burstTableBody = document.querySelector("#burst-table tbody");
const chaosFlagsDiv = document.getElementById("chaos-flags");
const healthStrip = document.getElementById("health-strip");
async function loadAccounts() {
const resp = await fetch("/api/accounts");
const accounts = await resp.json();
accountSelect.innerHTML = accounts
.slice(0, 200)
.map((a) => `<option value="${a.id}">${a.holder_name} (${a.currency})</option>`)
.join("");
}
txForm.addEventListener("submit", async (e) => {
e.preventDefault();
const payload = {
account_id: accountSelect.value,
kind: document.querySelector('input[name="kind"]:checked').value,
amount_minor: Number(document.getElementById("amount-input").value),
currency: "EUR",
merchant_name: document.getElementById("merchant-input").value,
};
const start = performance.now();
const resp = await fetch("/api/transactions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
const rtt = performance.now() - start;
const body = await resp.json();
resultRtt.textContent = rtt.toFixed(1);
resultRequestId.textContent = body.request_id || "--";
resultJson.textContent = JSON.stringify(body, null, 2);
});
function percentile(sorted, p) {
if (sorted.length === 0) return 0;
const rank = Math.max(1, Math.min(sorted.length, Math.ceil(p * sorted.length)));
return sorted[rank - 1];
}
burstButton.addEventListener("click", async () => {
const n = Number(burstN.value);
const accounts = Array.from(accountSelect.options).map((o) => o.value);
burstTableBody.innerHTML = `<tr><td colspan="8">running ${n} requests...</td></tr>`;
const latencies = [];
let success = 0;
let errors = 0;
await Promise.all(
Array.from({ length: n }, async () => {
const accountId = accounts[Math.floor(Math.random() * accounts.length)];
const start = performance.now();
try {
const resp = await fetch("/api/transactions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
account_id: accountId,
kind: "purchase",
amount_minor: 500,
currency: "EUR",
merchant_name: "Load Burst",
}),
});
latencies.push(performance.now() - start);
if (resp.ok) success += 1;
else errors += 1;
} catch (e) {
latencies.push(performance.now() - start);
errors += 1;
}
})
);
latencies.sort((a, b) => a - b);
const mean = latencies.reduce((a, b) => a + b, 0) / latencies.length;
const row = [
latencies.length,
success,
errors,
latencies[0].toFixed(1),
mean.toFixed(1),
percentile(latencies, 0.5).toFixed(1),
percentile(latencies, 0.95).toFixed(1),
latencies[latencies.length - 1].toFixed(1),
];
burstTableBody.innerHTML = `<tr>${row.map((v) => `<td>${v}</td>`).join("")}</tr>`;
});
async function loadChaos() {
const resp = await fetch("/api/admin/chaos");
const { flags } = await resp.json();
chaosFlagsDiv.innerHTML = Object.entries(flags)
.map(
([flag, enabled]) => `
<label>
<input type="checkbox" data-flag="${flag}" ${enabled ? "checked" : ""}>
${flag}
</label>`
)
.join("");
chaosFlagsDiv.querySelectorAll("input[type=checkbox]").forEach((el) => {
el.addEventListener("change", async () => {
await fetch("/api/admin/chaos", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ flag: el.dataset.flag, enabled: el.checked }),
});
});
});
}
document.querySelectorAll("#chaos-profiles button").forEach((btn) => {
btn.addEventListener("click", async () => {
await fetch("/api/admin/chaos/profile", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ profile: btn.dataset.profile }),
});
await loadChaos();
});
});
async function pollHealth() {
try {
const resp = await fetch("/api/health/all");
const { services } = await resp.json();
healthStrip.innerHTML = Object.entries(services)
.map(([name, status]) => `<span class="dot ${status === "ok" ? "green" : "red"}">${name}</span>`)
.join("");
} catch (e) {
// ignore transient polling failures
}
}
loadAccounts();
loadChaos();
pollHealth();
setInterval(pollHealth, 5000);
+80
View File
@@ -0,0 +1,80 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Ledgerline</title>
<link rel="stylesheet" href="/static/style.css">
</head>
<body>
<header>
<h1>Ledgerline</h1>
<p class="tagline">a deliberately imperfect payments backend</p>
</header>
<main>
<section id="submit-panel">
<h2>Submit a transaction</h2>
<form id="tx-form">
<label>Account
<select id="account-select"></select>
</label>
<label>Amount (minor units)
<input type="number" id="amount-input" value="1000" min="0">
</label>
<label>Merchant
<input type="text" id="merchant-input" value="Corner Store">
</label>
<fieldset>
<legend>Kind</legend>
<label><input type="radio" name="kind" value="deposit" checked> deposit</label>
<label><input type="radio" name="kind" value="withdrawal"> withdrawal</label>
<label><input type="radio" name="kind" value="purchase"> purchase</label>
</fieldset>
<button type="submit">Submit</button>
</form>
</section>
<section id="result-panel">
<h2>Result</h2>
<p>Round-trip: <span id="result-rtt">--</span> ms &middot; request_id: <code id="result-request-id">--</code></p>
<pre id="result-json">(nothing yet)</pre>
</section>
<section id="load-panel">
<h2>Load burst</h2>
<label>N
<select id="burst-n">
<option value="10">10</option>
<option value="50">50</option>
<option value="200">200</option>
</select>
</label>
<button id="burst-button">Fire burst</button>
<table id="burst-table">
<thead>
<tr><th>count</th><th>success</th><th>errors</th><th>min</th><th>mean</th><th>p50</th><th>p95</th><th>max</th></tr>
</thead>
<tbody><tr><td colspan="8">no burst run yet</td></tr></tbody>
</table>
</section>
<section id="chaos-panel">
<h2>Chaos</h2>
<div id="chaos-flags"></div>
<div id="chaos-profiles">
<button data-profile="clean">clean</button>
<button data-profile="light">light</button>
<button data-profile="nightmare">nightmare</button>
<button data-profile="random">random</button>
</div>
</section>
<section id="health-panel">
<h2>Service health</h2>
<div id="health-strip"></div>
</section>
</main>
<script src="/static/app.js"></script>
</body>
</html>
+184
View File
@@ -0,0 +1,184 @@
:root {
color-scheme: dark;
--bg: #0f1117;
--panel: #171a23;
--border: #2a2e3a;
--text: #d8dce6;
--muted: #7b8296;
--accent: #5fd0a8;
--danger: #e0685f;
--mono: "SF Mono", "Fira Code", Consolas, monospace;
}
* {
box-sizing: border-box;
}
body {
background: var(--bg);
color: var(--text);
font-family: var(--mono);
margin: 0;
padding: 0 1.5rem 3rem;
}
header {
padding: 2rem 0 1rem;
border-bottom: 1px solid var(--border);
}
header h1 {
margin: 0;
font-size: 1.6rem;
letter-spacing: 0.02em;
}
.tagline {
color: var(--muted);
margin: 0.25rem 0 0;
font-size: 0.85rem;
}
main {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
gap: 1.5rem;
margin-top: 1.5rem;
}
section {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 6px;
padding: 1rem 1.25rem;
}
section h2 {
margin-top: 0;
font-size: 1rem;
color: var(--accent);
text-transform: uppercase;
letter-spacing: 0.05em;
}
label {
display: block;
margin-bottom: 0.6rem;
font-size: 0.85rem;
color: var(--muted);
}
input, select, button {
font-family: var(--mono);
background: #0c0e13;
color: var(--text);
border: 1px solid var(--border);
border-radius: 4px;
padding: 0.4rem 0.5rem;
margin-top: 0.2rem;
width: 100%;
}
fieldset {
border: 1px solid var(--border);
border-radius: 4px;
margin-bottom: 0.6rem;
}
fieldset label {
display: inline-block;
width: auto;
margin-right: 1rem;
}
button {
cursor: pointer;
background: var(--accent);
color: #0c0e13;
font-weight: 600;
border: none;
margin-top: 0.5rem;
transition: opacity 0.15s ease;
}
button:hover {
opacity: 0.85;
}
#chaos-profiles {
display: flex;
gap: 0.5rem;
margin-top: 0.75rem;
}
#chaos-profiles button {
width: auto;
flex: 1;
}
pre#result-json {
background: #0c0e13;
border: 1px solid var(--border);
border-radius: 4px;
padding: 0.75rem;
overflow-x: auto;
font-size: 0.8rem;
max-height: 260px;
}
code {
background: #0c0e13;
padding: 0.1rem 0.3rem;
border-radius: 3px;
user-select: all;
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 0.75rem;
font-size: 0.8rem;
}
th, td {
border: 1px solid var(--border);
padding: 0.35rem 0.5rem;
text-align: right;
}
th:first-child, td:first-child {
text-align: left;
}
#chaos-flags label {
display: flex;
align-items: center;
gap: 0.4rem;
}
#chaos-flags input {
width: auto;
}
#health-strip {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
}
.dot {
border-radius: 999px;
padding: 0.25rem 0.6rem;
font-size: 0.75rem;
border: 1px solid var(--border);
}
.dot.green {
color: var(--accent);
border-color: var(--accent);
}
.dot.red {
color: var(--danger);
border-color: var(--danger);
}