Add Phase 0a black-box integration test suite
81 tests across 11 files plus a 5-module harness (load, pg, streams, containers, payloads), covering all eleven Phase 0 goals and all 13 named chaos-flag pathologies. Talks to the system only over HTTP, Postgres, Redis, and the Docker CLI -- no imports of application code. RED-BASELINE.md records the suite failing red against the empty repo before any implementation existed; README.md documents the request-contract fields invented to fill gaps in the spec, measurement methodology, and known-flaky tests. This suite is a frozen contract from this point forward per CLAUDE.md.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
"""G8 -- nightmare mode degrades but does not collapse."""
|
||||
import random
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from harness.config import GATEWAY_URL
|
||||
from harness.load import drive
|
||||
|
||||
pytestmark = pytest.mark.pathology
|
||||
|
||||
|
||||
async def test_nightmare_degrades_but_survives(chaos, manifest, api):
|
||||
"""G8: same driver, chaos-clean then chaos-nightmare, 300 requests at
|
||||
concurrency 20 each. Nightmare p95 >= 5x clean p95; nightmare error rate
|
||||
in [2%, 15%]; >= 80% of requests still return a valid envelope (success or
|
||||
a structured error, never a connection reset or empty body);
|
||||
GET /api/health/all still responds within 5s during the burst."""
|
||||
accounts = manifest["cold_account_ids_sample"] + manifest["hot_account_ids"]
|
||||
|
||||
def url_factory():
|
||||
account_id = random.choice(accounts)
|
||||
return (f"{GATEWAY_URL}/api/accounts/{account_id}/insights", {})
|
||||
|
||||
await chaos.set_profile("clean")
|
||||
clean_report = await drive("GET", url_factory, total=300, concurrency=20, warmup=20)
|
||||
|
||||
await chaos.set_profile("nightmare")
|
||||
nightmare_report = await drive("GET", url_factory, total=300, concurrency=20, warmup=20)
|
||||
|
||||
assert nightmare_report.p95 >= 5 * clean_report.p95, (
|
||||
f"nightmare p95 not >= 5x clean p95\nclean={clean_report}\nnightmare={nightmare_report}"
|
||||
)
|
||||
assert 0.02 <= nightmare_report.error_rate <= 0.15, str(nightmare_report)
|
||||
|
||||
valid_requests = nightmare_report.count - nightmare_report.exceptions
|
||||
valid_share = valid_requests / nightmare_report.count
|
||||
assert valid_share >= 0.80, (
|
||||
f"only {valid_requests}/{nightmare_report.count} requests returned a "
|
||||
f"valid envelope (no reset/empty body): {nightmare_report}"
|
||||
)
|
||||
|
||||
start = time.perf_counter()
|
||||
resp = await api.get("/api/health/all")
|
||||
elapsed = time.perf_counter() - start
|
||||
assert elapsed < 5, f"/api/health/all took {elapsed:.2f}s during the nightmare burst"
|
||||
assert resp.status_code == 200
|
||||
Reference in New Issue
Block a user