Add Phase 0a black-box integration test suite

81 tests across 11 files plus a 5-module harness (load, pg, streams,
containers, payloads), covering all eleven Phase 0 goals and all 13 named
chaos-flag pathologies. Talks to the system only over HTTP, Postgres, Redis,
and the Docker CLI -- no imports of application code. RED-BASELINE.md records
the suite failing red against the empty repo before any implementation
existed; README.md documents the request-contract fields invented to fill
gaps in the spec, measurement methodology, and known-flaky tests.

This suite is a frozen contract from this point forward per CLAUDE.md.

tests/harness/config.py

@@ -0,0 +1,43 @@
+"""Environment-driven configuration shared by tests and harness helpers.
+
+Every value here can be overridden via env var; the defaults match the ports
+documented in the project READMEs (gateway 8000, ledger..notifier 8001-8005,
+postgres 5432, redis 6379). No test file should hardcode a port or DSN --
+import it from here instead.
+"""
+import os
+from pathlib import Path
+
+# tests/harness/config.py -> parents[0]=harness, [1]=tests, [2]=repo root
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+GATEWAY_URL = os.environ.get("GATEWAY_URL", "http://localhost:8000")
+SERVICE_URLS = {
+    "gateway": GATEWAY_URL,
+    "ledger": os.environ.get("LEDGER_URL", "http://localhost:8001"),
+    "fraud": os.environ.get("FRAUD_URL", "http://localhost:8002"),
+    "sanctions": os.environ.get("SANCTIONS_URL", "http://localhost:8003"),
+    "insights": os.environ.get("INSIGHTS_URL", "http://localhost:8004"),
+    "notifier": os.environ.get("NOTIFIER_URL", "http://localhost:8005"),
+}
+HTTP_SERVICES = tuple(SERVICE_URLS.keys())
+
+POSTGRES_DSN = os.environ.get(
+    "POSTGRES_DSN", "postgresql://ledgerline:ledgerline@localhost:5432/ledgerline"
+)
+REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+
+STREAM_NAME = os.environ.get("LEDGERLINE_STREAM", "stream:transactions")
+CONSUMER_GROUPS = ("notifier", "audit")
+
+DEFAULT_HTTP_TIMEOUT = float(os.environ.get("LEDGERLINE_HTTP_TIMEOUT", "15.0"))
+
+SEED = int(os.environ.get("SEED", "42"))
+
+# Assumption documented in tests/README.md under "Log-line assumptions": the
+# sanctions service logs a line containing "lookup" for every watchlist check
+# it performs, and every service logs a line containing "healthz" when its
+# health endpoint is hit. Agent B's implementation must satisfy these for the
+# 6.8/6.9/6.14 pathology tests to be meaningful.
+SANCTIONS_LOOKUP_LOG_PATTERN = r"(?i)lookup"
+HEALTHZ_LOG_PATTERN = r"(?i)healthz"

tests/harness/containers.py

@@ -0,0 +1,104 @@
+"""docker stats / docker compose logs / container lifecycle helpers.
+
+Per phase-0a-test-suite-spec.md §7. All `docker compose` invocations run with
+`cwd=REPO_ROOT` so they work regardless of the pytest invocation's own cwd.
+"""
+import json
+import re
+import subprocess
+import time
+from datetime import datetime
+
+from .config import REPO_ROOT
+
+
+def _run(cmd, timeout=30):
+    return subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True, timeout=timeout)
+
+
+def _compose_ps(service):
+    result = _run(["docker", "compose", "ps", service, "--format", "json"])
+    lines = [l for l in result.stdout.strip().splitlines() if l.strip()]
+    if not lines:
+        return None
+    return json.loads(lines[0])
+
+
+def stop(service):
+    result = _run(["docker", "compose", "stop", service])
+    if result.returncode != 0:
+        raise RuntimeError(f"docker compose stop {service} failed: {result.stderr}")
+
+
+def start(service):
+    result = _run(["docker", "compose", "start", service])
+    if result.returncode != 0:
+        raise RuntimeError(f"docker compose start {service} failed: {result.stderr}")
+
+
+def wait_ready(service, timeout=30.0):
+    """Poll `docker compose ps` until the container reports healthy (or, absent
+    a healthcheck, running). Raises TimeoutError otherwise."""
+    deadline = time.monotonic() + timeout
+    last_state = None
+    while time.monotonic() < deadline:
+        info = _compose_ps(service)
+        if info:
+            health = info.get("Health", "")
+            state = info.get("State", "")
+            last_state = (state, health)
+            if health == "healthy" or (health == "" and state == "running"):
+                return
+        time.sleep(1)
+    raise TimeoutError(f"{service} not ready within {timeout}s (last state={last_state})")
+
+
+_MEM_UNITS = {
+    "B": 1 / (1024 * 1024),
+    "KIB": 1 / 1024,
+    "KB": 1 / 1024,
+    "MIB": 1,
+    "MB": 1,
+    "GIB": 1024,
+    "GB": 1024,
+    "TIB": 1024 * 1024,
+    "TB": 1024 * 1024,
+}
+
+
+def _parse_mem_to_mb(s):
+    m = re.match(r"([\d.]+)\s*([A-Za-z]+)", s.strip())
+    if not m:
+        raise ValueError(f"cannot parse memory string: {s!r}")
+    value, unit = float(m.group(1)), m.group(2).upper()
+    if unit not in _MEM_UNITS:
+        raise ValueError(f"unknown memory unit {unit!r} in {s!r}")
+    return value * _MEM_UNITS[unit]
+
+
+def rss_mb(service):
+    """Current RSS (approximated by `docker stats` MemUsage) of `service`'s container, in MB."""
+    info = _compose_ps(service)
+    if not info:
+        raise RuntimeError(f"container for {service!r} not found -- is the stack up?")
+    container_id = info.get("ID") or info.get("Id")
+    result = _run(["docker", "stats", container_id, "--no-stream", "--format", "{{.MemUsage}}"])
+    mem_str = result.stdout.strip().split("/")[0].strip()
+    return _parse_mem_to_mb(mem_str)
+
+
+def count_log_lines(service, pattern, since):
+    """Count lines in `docker compose logs <service>` since `since` matching `pattern`.
+
+    `since` is a timezone-aware datetime or a docker-compatible string (RFC3339
+    or a relative duration like "30s"). This is deliberately the only place
+    that shells out to `docker compose logs` for counting -- see
+    tests/README.md, "Log-counting migration", for why and what replaces it
+    from Phase 3 onward.
+    """
+    since_arg = since.isoformat() if isinstance(since, datetime) else since
+    result = _run(
+        ["docker", "compose", "logs", service, "--since", since_arg, "--no-color", "--no-log-prefix"]
+    )
+    regex = re.compile(pattern)
+    return sum(1 for line in result.stdout.splitlines() if regex.search(line))

tests/harness/load.py

@@ -0,0 +1,133 @@
+"""Concurrent request driver + LatencyReport, per phase-0a-test-suite-spec.md §7.
+
+A closed-loop driver (fixed concurrency) is used deliberately for its
+simplicity here. This under-measures latency under saturation (coordinated
+omission) -- Phase 6 replaces it with k6's arrival-rate executors. Don't try
+to fix that in this suite; note it, live with it.
+"""
+import asyncio
+import math
+import statistics
+import time
+from dataclasses import dataclass, field
+from typing import Callable
+
+import httpx
+
+
+@dataclass
+class LatencyReport:
+    count: int
+    ok: int
+    http_errors: int  # non-2xx HTTP responses
+    exceptions: int  # requests that raised (timeout, connection reset, ...)
+    errors: int  # http_errors + exceptions
+    error_rate: float
+    status_counts: dict
+    p50: float
+    p95: float
+    p99: float
+    max: float
+    throughput_rps: float
+    wall_seconds: float
+    latencies_ms: list = field(default_factory=list, repr=False)
+
+    def __str__(self):
+        return (
+            f"n={self.count} ok={self.ok} http_errors={self.http_errors} "
+            f"exceptions={self.exceptions} error_rate={self.error_rate:.1%} "
+            f"p50={self.p50:.1f}ms p95={self.p95:.1f}ms p99={self.p99:.1f}ms "
+            f"max={self.max:.1f}ms throughput={self.throughput_rps:.1f}rps "
+            f"wall={self.wall_seconds:.2f}s status_counts={self.status_counts}"
+        )
+
+
+def _nearest_rank(sorted_vals, p):
+    """Nearest-rank percentile: no interpolation, no averaging."""
+    n = len(sorted_vals)
+    if n == 0:
+        return 0.0
+    rank = max(1, min(n, math.ceil(p * n)))
+    return sorted_vals[rank - 1]
+
+
+async def drive(
+    method: str,
+    url_factory: Callable[[], tuple],
+    total: int,
+    concurrency: int,
+    warmup: int = 20,
+    timeout: float = 15.0,
+) -> LatencyReport:
+    """Drive `total` requests at fixed `concurrency`, discarding `warmup` first.
+
+    `url_factory()` returns `(url, kwargs)`, called once per request so a test
+    can vary the target account/params per call.
+    """
+    async with httpx.AsyncClient(timeout=timeout) as client:
+
+        async def one():
+            url, kwargs = url_factory()
+            t0 = time.perf_counter()
+            try:
+                resp = await client.request(method, url, **kwargs)
+                return (time.perf_counter() - t0) * 1000, resp.status_code
+            except Exception:
+                return (time.perf_counter() - t0) * 1000, None
+
+        sem = asyncio.Semaphore(concurrency)
+
+        async def bound(collector):
+            async with sem:
+                result = await one()
+                if collector is not None:
+                    collector.append(result)
+
+        if warmup:
+            await asyncio.gather(*[bound(None) for _ in range(warmup)])
+
+        results: list = []
+        wall_start = time.perf_counter()
+        await asyncio.gather(*[bound(results) for _ in range(total)])
+        wall_seconds = time.perf_counter() - wall_start
+
+    latencies = sorted(r[0] for r in results)
+    status_counts: dict = {}
+    ok = http_errors = exceptions = 0
+    for _, status in results:
+        if status is None:
+            exceptions += 1
+            status_counts["exception"] = status_counts.get("exception", 0) + 1
+        elif 200 <= status < 300:
+            ok += 1
+            status_counts[status] = status_counts.get(status, 0) + 1
+        else:
+            http_errors += 1
+            status_counts[status] = status_counts.get(status, 0) + 1
+
+    count = len(results)
+    errors = http_errors + exceptions
+    return LatencyReport(
+        count=count,
+        ok=ok,
+        http_errors=http_errors,
+        exceptions=exceptions,
+        errors=errors,
+        error_rate=(errors / count if count else 0.0),
+        status_counts=status_counts,
+        p50=_nearest_rank(latencies, 0.50),
+        p95=_nearest_rank(latencies, 0.95),
+        p99=_nearest_rank(latencies, 0.99),
+        max=(latencies[-1] if latencies else 0.0),
+        throughput_rps=(count / wall_seconds if wall_seconds > 0 else 0.0),
+        wall_seconds=wall_seconds,
+        latencies_ms=latencies,
+    )
+
+
+async def measure_median(n: int, fn: Callable, key=lambda x: x):
+    """Run async `fn()` n times and return the run whose `key(result)` is the
+    median -- e.g. key=lambda r: r.p95 picks the median-p95 LatencyReport
+    whole, rather than averaging percentiles across runs (never do that)."""
+    results = [await fn() for _ in range(n)]
+    return sorted(results, key=key)[len(results) // 2]

tests/harness/payloads.py

@@ -0,0 +1,28 @@
+"""The POST /api/transactions request shape this suite assumes.
+
+Neither spec pins down the request field names for creating a transaction --
+only the response envelope (phase-0-base-project-spec.md §6.1) is specified.
+Since the test suite is the contract, this module fixes the shape Agent B's
+implementation must accept. See tests/README.md, "Request contract
+assumptions", for the rationale.
+"""
+
+
+def tx_payload(
+    account_id,
+    kind="deposit",
+    amount_minor=1000,
+    currency="EUR",
+    merchant_name="Test Merchant",
+    category_id=None,
+):
+    payload = {
+        "account_id": account_id,
+        "kind": kind,
+        "amount_minor": amount_minor,
+        "currency": currency,
+        "merchant_name": merchant_name,
+    }
+    if category_id is not None:
+        payload["category_id"] = category_id
+    return payload

tests/harness/pg.py

@@ -0,0 +1,74 @@
+"""pg_stat_statements deltas and EXPLAIN helpers, per phase-0a-test-suite-spec.md §7."""
+from contextlib import contextmanager
+
+import psycopg
+
+
+class _Counter:
+    """Populated with `.calls` (and `.total_time_ms`) once the `with` block exits."""
+
+    def __init__(self):
+        self.calls = None
+        self.total_time_ms = None
+
+    def __repr__(self):
+        return f"Counter(calls={self.calls}, total_time_ms={self.total_time_ms})"
+
+    def __str__(self):
+        return repr(self)
+
+
+@contextmanager
+def statement_count(dsn):
+    """Reset pg_stat_statements, run the caller's block, yield the call-count delta.
+
+    pg_stat_statements is reset globally, so this is only meaningful when
+    nothing else is hitting the database concurrently during the `with` block
+    -- true for the single-request pathology assertions this is built for.
+    """
+    counter = _Counter()
+    with psycopg.connect(dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT pg_stat_statements_reset()")
+        try:
+            yield counter
+        finally:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "SELECT COALESCE(SUM(calls), 0), COALESCE(SUM(total_exec_time), 0) "
+                    "FROM pg_stat_statements"
+                )
+                calls, total_time_ms = cur.fetchone()
+            counter.calls = calls
+            counter.total_time_ms = total_time_ms
+
+
+def explain(dsn, sql, params=None):
+    """Run EXPLAIN (ANALYZE, FORMAT JSON) and return the plan dict."""
+    with psycopg.connect(dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            cur.execute(f"EXPLAIN (ANALYZE, FORMAT JSON) {sql}", params or ())
+            (plan,) = cur.fetchone()
+            return plan[0]
+
+
+def plan_node_types(plan: dict) -> set:
+    """Flatten an EXPLAIN JSON plan (or its top-level {"Plan": ...} wrapper)
+    into the set of "Node Type" values it contains, at any depth."""
+    types: set = set()
+    root = plan.get("Plan", plan) if isinstance(plan, dict) else plan
+
+    def walk(node):
+        if isinstance(node, dict):
+            if "Node Type" in node:
+                types.add(node["Node Type"])
+            for key, value in node.items():
+                if key == "Plans" and isinstance(value, list):
+                    for child in value:
+                        walk(child)
+        elif isinstance(node, list):
+            for item in node:
+                walk(item)
+
+    walk(root)
+    return types

tests/harness/streams.py

@@ -0,0 +1,28 @@
+"""Redis XINFO GROUPS helpers, per phase-0a-test-suite-spec.md §7."""
+import time
+
+
+def group_lag(redis_client, stream: str, group: str) -> dict:
+    """Return the XINFO GROUPS entry for `group` on `stream` as a dict."""
+    groups = redis_client.xinfo_groups(stream)
+    for g in groups:
+        if g.get("name") == group:
+            return g
+    raise ValueError(f"consumer group {group!r} not found on stream {stream!r}: {groups}")
+
+
+def wait_for_drain(redis_client, stream: str, group: str, timeout: float = 15.0) -> None:
+    """Block until `group`'s lag and pending count both reach 0, or raise TimeoutError."""
+    deadline = time.monotonic() + timeout
+    last = None
+    while time.monotonic() < deadline:
+        g = group_lag(redis_client, stream, group)
+        lag = g.get("lag") or 0
+        pending = g.get("pending") or 0
+        last = (lag, pending)
+        if lag == 0 and pending == 0:
+            return
+        time.sleep(0.25)
+    raise TimeoutError(
+        f"group {group!r} on {stream!r} did not drain within {timeout}s (last lag/pending={last})"
+    )