Add Phase 0a black-box integration test suite
81 tests across 11 files plus a 5-module harness (load, pg, streams, containers, payloads), covering all eleven Phase 0 goals and all 13 named chaos-flag pathologies. Talks to the system only over HTTP, Postgres, Redis, and the Docker CLI -- no imports of application code. RED-BASELINE.md records the suite failing red against the empty repo before any implementation existed; README.md documents the request-contract fields invented to fill gaps in the spec, measurement methodology, and known-flaky tests. This suite is a frozen contract from this point forward per CLAUDE.md.
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
"""G10 -- no observability tooling is present yet.
|
||||
|
||||
This test module guards the Phase 0 "before" state. It is expected to be
|
||||
DELETED at the start of Phase 2, not maintained -- once real instrumentation
|
||||
is added, every assertion here becomes actively wrong on purpose.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from harness.config import HTTP_SERVICES, REPO_ROOT, SERVICE_URLS
|
||||
|
||||
pytestmark = pytest.mark.guard
|
||||
|
||||
FORBIDDEN_IMPORTS = ("opentelemetry", "prometheus_client", "structlog", "sentry_sdk", "elasticapm")
|
||||
|
||||
|
||||
def _python_files():
|
||||
files = []
|
||||
for name in ("services", "common"):
|
||||
base = REPO_ROOT / name
|
||||
if base.is_dir():
|
||||
files.extend(base.rglob("*.py"))
|
||||
return files
|
||||
|
||||
|
||||
def test_application_code_exists():
|
||||
"""G10 (vacuous-pass guard): this module must not pass vacuously against
|
||||
an empty repo -- there must be application code for the import check
|
||||
below to actually inspect."""
|
||||
files = _python_files()
|
||||
assert files, (
|
||||
"no .py files found under services/ or common/ -- either the app hasn't "
|
||||
"been implemented yet, or the guard tests below would pass vacuously"
|
||||
)
|
||||
|
||||
|
||||
def test_no_file_imports_observability_libraries():
|
||||
"""G10: no file under services/ or common/ imports an observability library."""
|
||||
offenders = []
|
||||
for path in _python_files():
|
||||
text = path.read_text(errors="ignore")
|
||||
for lib in FORBIDDEN_IMPORTS:
|
||||
if re.search(rf"^\s*(import|from)\s+{re.escape(lib)}\b", text, re.MULTILINE):
|
||||
offenders.append((str(path.relative_to(REPO_ROOT)), lib))
|
||||
assert not offenders, offenders
|
||||
|
||||
|
||||
def test_dependency_files_do_not_list_observability_libraries():
|
||||
"""G10: neither requirements*.txt (excluding requirements-test.txt) nor
|
||||
pyproject.toml lists an observability library."""
|
||||
candidates = [p for p in REPO_ROOT.glob("requirements*.txt") if p.name != "requirements-test.txt"]
|
||||
pyproject = REPO_ROOT / "pyproject.toml"
|
||||
if pyproject.is_file():
|
||||
candidates.append(pyproject)
|
||||
|
||||
offenders = []
|
||||
for path in candidates:
|
||||
text = path.read_text(errors="ignore").lower()
|
||||
for lib in FORBIDDEN_IMPORTS:
|
||||
if lib in text:
|
||||
offenders.append((str(path.relative_to(REPO_ROOT)), lib))
|
||||
assert not offenders, offenders
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service", HTTP_SERVICES)
|
||||
async def test_no_service_exposes_metrics_endpoint(service):
|
||||
"""G10: no service exposes /metrics."""
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
resp = await client.get(f"{SERVICE_URLS[service]}/metrics")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
async def test_gateway_logs_are_plain_text_not_json():
|
||||
"""G10: docker compose logs gateway emits plain text, not JSON: sample 20
|
||||
lines and assert none parse as a JSON object."""
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
for _ in range(3):
|
||||
await client.get(f"{SERVICE_URLS['gateway']}/healthz")
|
||||
|
||||
since = (datetime.now(timezone.utc) - timedelta(minutes=10)).isoformat()
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "logs", "gateway", "--since", since, "--no-color", "--no-log-prefix", "--tail", "20"],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
lines = [l for l in result.stdout.splitlines() if l.strip()][-20:]
|
||||
assert lines, "no recent gateway log lines found"
|
||||
|
||||
json_lines = 0
|
||||
for line in lines:
|
||||
try:
|
||||
json.loads(line)
|
||||
json_lines += 1
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
assert json_lines == 0, f"{json_lines}/{len(lines)} gateway log lines parsed as JSON"
|
||||
Reference in New Issue
Block a user