Add Phase 0a black-box integration test suite
81 tests across 11 files plus a 5-module harness (load, pg, streams, containers, payloads), covering all eleven Phase 0 goals and all 13 named chaos-flag pathologies. Talks to the system only over HTTP, Postgres, Redis, and the Docker CLI -- no imports of application code. RED-BASELINE.md records the suite failing red against the empty repo before any implementation existed; README.md documents the request-contract fields invented to fill gaps in the spec, measurement methodology, and known-flaky tests. This suite is a frozen contract from this point forward per CLAUDE.md.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""G7 -- the clean baseline is actually healthy; otherwise G6 means nothing."""
|
||||
import random
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from harness.config import GATEWAY_URL, POSTGRES_DSN, STREAM_NAME
|
||||
from harness.load import drive
|
||||
from harness.payloads import tx_payload
|
||||
from harness.pg import statement_count
|
||||
from harness.streams import wait_for_drain
|
||||
|
||||
pytestmark = pytest.mark.baseline
|
||||
|
||||
|
||||
async def test_clean_baseline_is_healthy(chaos, manifest):
|
||||
"""G7: with chaos-clean, after 20 discarded warm-up requests, 200 requests
|
||||
at concurrency 10 against random cold accounts show zero errors, p95 <
|
||||
400ms (the one absolute threshold in the functional set -- if this fails on
|
||||
a slow machine, that machine cannot run the pathology suite meaningfully
|
||||
either), and p99/p50 < 5 (no fat tail)."""
|
||||
await chaos.set_profile("clean")
|
||||
cold_accounts = manifest["cold_account_ids_sample"]
|
||||
|
||||
def url_factory():
|
||||
account_id = random.choice(cold_accounts)
|
||||
return (f"{GATEWAY_URL}/api/accounts/{account_id}/insights", {})
|
||||
|
||||
report = await drive("GET", url_factory, total=200, concurrency=10, warmup=20)
|
||||
|
||||
assert report.error_rate == 0, (
|
||||
f"clean baseline must have zero errors -- got {report}. If this fails "
|
||||
f"on a slow machine, that machine cannot run the pathology suite "
|
||||
f"meaningfully either."
|
||||
)
|
||||
assert report.p95 < 400, str(report)
|
||||
assert report.p99 / report.p50 < 5, str(report)
|
||||
|
||||
|
||||
async def test_clean_insights_issues_few_statements(chaos, manifest):
|
||||
"""G7: insights for a cold account issues fewer than 10 SQL statements."""
|
||||
await chaos.set_profile("clean")
|
||||
account_id = manifest["cold_account_ids_sample"][0]
|
||||
|
||||
with statement_count(POSTGRES_DSN) as counter:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(f"{GATEWAY_URL}/api/accounts/{account_id}/insights")
|
||||
assert resp.status_code == 200
|
||||
|
||||
assert counter.calls < 10, f"insights issued {counter.calls} statements: {counter}"
|
||||
|
||||
|
||||
async def test_clean_redis_consumer_lag_drains_quickly(chaos, manifest, redis_client, fresh_balances):
|
||||
"""G7: Redis consumer lag returns to 0 within 15s of a burst finishing."""
|
||||
await chaos.set_profile("clean")
|
||||
account_id = manifest["clean_account_id"]
|
||||
fresh_balances(account_id)
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for _ in range(20):
|
||||
resp = await client.post(
|
||||
f"{GATEWAY_URL}/api/transactions",
|
||||
json=tx_payload(account_id, amount_minor=100, merchant_name="Baseline Burst"),
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
|
||||
for group in ("notifier", "audit"):
|
||||
wait_for_drain(redis_client, STREAM_NAME, group, timeout=15)
|
||||
Reference in New Issue
Block a user