ayushbharadva/patchpilot-api

1#!/usr/bin/env python32"""Loop timing harness + demo script of record (DEMO-03 / D-01 / D-02).3 4This is the demo script of record. It drives the RUNNING backend over real5HTTP (the cognee package is never imported here — this measures the real6network path a browser takes, per `frontend/lib/api.ts`'s exact endpoint7payloads), timing the full search -> release upload -> drift badge ->8forget -> re-search loop and asserting it lands under 120 seconds9(DEMO-03).10 11Canonical query: the SendGrid/forgot-password arc's flip query12(seed/README.md B-02).13 14SNAPSHOT REDESIGN (Jul 5, demo-dataset validation): the reset snapshot is15now the PRE-RELEASE state — `incidents` (🟢) + `workarounds_v1_8` (🟢)16only, `workarounds_v1_9` absent. This makes the timed loop the *literal*17Core Value arc with a TRUE answer flip: the initial search answers with the18old `flush_mail_queue` workaround (v1_8 is the highest live version, so it19is stable and wins as primary); uploading the real20`seed/workarounds_v1_9/sendgrid-release.md` as release 1.9 flips v1_8 to21drifting live; forgetting v1_8 flips the re-search answer to the SendGrid22API fix. No fabricated release note is needed anymore — the harness drives23the exact same documents and datasets the recorded demo uses. The trailing24`POST /reset` restores the canonical pre-release snapshot regardless of25what this run created.26 27NINTH DEVIATION (Rule 1 bug, fixed in `backend/reset.py`, not this script):28`POST /reset` was failing with `{"status":"error"}` before this harness29could even take its first `/reset` reading — cognee's session/feedback30cache layer (`CACHING=true`) holds its own SQLite handle31(`.patchpilot_memory/databases/cache.db`) that `backend/reset.py`'s32Windows-safe handle-release sequence never closed, so33`scripts/snapshot_memory.restore()`'s `shutil.rmtree()` raised34`PermissionError: [WinError 32]`. Fixed by awaiting cognee's own35`close_cache_engine()` helper alongside the existing relational/vector/graph36releases. See `backend/reset.py`'s module docstring for the full trace.37 38Usage:39    .venv/Scripts/python.exe scripts/time_demo_loop.py40"""41 42import sys43import time44 45import requests46 47BASE_URL = "http://localhost:8000"48QUERY = "What is the fix for forgot password emails not sending?"49BUDGET_SECONDS = 120.050 51# The REAL v1.9 release note (seed/README.md's before/after arc) — uploaded52# live so the harness exercises the exact ingest -> drift -> forget ->53# re-search loop the recorded demo performs, with the same document.54from pathlib import Path  # noqa: E40255 56_RELEASE_1_9_PATH = (57    Path(__file__).resolve().parent.parent / "seed" / "workarounds_v1_9" / "sendgrid-release.md"58)59RELEASE_1_9_CONTENT = _RELEASE_1_9_PATH.read_text()60 61INGEST_POLL_INTERVAL_SECONDS = 1.062INGEST_POLL_TIMEOUT_SECONDS = 90.063 64 65class LoopFailure(RuntimeError):66    """Raised with the failing step name so failures are unambiguous."""67 68 69def _post_json(path: str, payload: dict) -> dict:70    resp = requests.post(f"{BASE_URL}{path}", json=payload, timeout=30)71    resp.raise_for_status()72    return resp.json()73 74 75def _get_json(path: str, params: dict | None = None) -> dict | list:76    resp = requests.get(f"{BASE_URL}{path}", params=params, timeout=30)77    resp.raise_for_status()78    return resp.json()79 80 81def _timed_step(name: str, fn):82    start = time.perf_counter()83    result = fn()84    elapsed = time.perf_counter() - start85    print(f"{name}: {elapsed:.1f}s")86    return result, elapsed87 88 89def step_reset() -> None:90    data = _post_json("/reset", {})91    if data.get("status") != "reset":92        raise LoopFailure(f"reset: expected status=reset, got {data!r}")93 94 95def step_search_initial() -> dict:96    data = _post_json("/search", {"query": QUERY})97    if data.get("status") != "ok":98        raise LoopFailure(f"search (initial): expected status=ok, got {data!r}")99    return data100 101 102def step_upload_release() -> None:103    files = [("files", ("sendgrid-release.md", RELEASE_1_9_CONTENT, "text/markdown"))]104    form = {"content_type": "release_note", "release_version": "1_9"}105    resp = requests.post(f"{BASE_URL}/ingest", files=files, data=form, timeout=30)106    resp.raise_for_status()107    data = resp.json()108    if data.get("status") != "accepted":109        raise LoopFailure(f"ingest: expected status=accepted, got {data!r}")110 111    deadline = time.perf_counter() + INGEST_POLL_TIMEOUT_SECONDS112    while time.perf_counter() < deadline:113        status_data = _get_json("/ingest/status", {"dataset": "workarounds_v1_9"})114        if status_data.get("status") == "ready":115            return116        if status_data.get("status") == "failed":117            raise LoopFailure(f"ingest/status: dataset failed to process: {status_data!r}")118        time.sleep(INGEST_POLL_INTERVAL_SECONDS)119    raise LoopFailure("ingest/status: timed out waiting for workarounds_v1_9 to become ready")120 121 122def step_confirm_drift() -> None:123    datasets = _get_json("/datasets")124    by_name = {d["name"]: d for d in datasets}125    v1_8 = by_name.get("workarounds_v1_8")126    if v1_8 is None:127        raise LoopFailure("datasets: workarounds_v1_8 missing after release upload")128    if v1_8.get("drift_state") != "drifting":129        raise LoopFailure(130            f"datasets: expected workarounds_v1_8 drift_state=drifting after v1_9 "131            f"upload, got {v1_8.get('drift_state')!r}"132        )133    if not v1_8.get("drift_reason"):134        raise LoopFailure("datasets: workarounds_v1_8 is drifting but has no drift_reason")135 136 137def step_forget() -> None:138    data = _post_json("/forget", {"dataset": "workarounds_v1_8"})139    if data.get("status") != "forgotten":140        raise LoopFailure(f"forget: expected status=forgotten, got {data!r}")141 142 143def step_search_after_forget() -> dict:144    data = _post_json("/search", {"query": QUERY})145    if data.get("status") != "ok":146        raise LoopFailure(147            f"search (after forget): expected status=ok (B-01 -- re-search must survive "148            f"forget), got {data!r}"149        )150    datasets = _get_json("/datasets")151    names = {d["name"] for d in datasets}152    if "workarounds_v1_8" in names:153        raise LoopFailure("datasets: workarounds_v1_8 still present after forget")154    return data155 156 157def step_confirm_graph() -> dict:158    data = _get_json("/graph")159    nodes = data.get("nodes") if isinstance(data, dict) else None160    if not nodes:161        raise LoopFailure(f"graph: expected non-empty nodes, got {data!r}")162    return data163 164 165def main() -> int:166    print(f"PatchPilot demo loop timing harness -- query: {QUERY!r}")167 168    try:169        # Setup -- not counted in the DEMO-03 loop budget (demo operator's170        # one-click reset before the timed run starts).171        _, reset_elapsed = _timed_step("setup: reset", step_reset)172        print(f"  (setup reset not counted toward the {BUDGET_SECONDS:.0f}s budget)")173 174        loop_start = time.perf_counter()175 176        before, t1 = _timed_step("1: search (before release)", step_search_initial)177        print(f"  root_cause[:80]={before.get('root_cause', '')[:80]!r} "178              f"source_dataset={before.get('source_dataset')!r}")179 180        _, t2 = _timed_step("2: ingest release + poll until ready", step_upload_release)181 182        _, t3 = _timed_step("3: confirm drift badge", step_confirm_drift)183 184        _, t4 = _timed_step("4: forget", step_forget)185 186        after, t5 = _timed_step("5: re-search (after forget)", step_search_after_forget)187        print(f"  root_cause[:80]={after.get('root_cause', '')[:80]!r} "188              f"source_dataset={after.get('source_dataset')!r}")189 190        loop_total = time.perf_counter() - loop_start191        print(f"TOTAL: {loop_total:.1f}s (budget {BUDGET_SECONDS:.0f}s)")192 193        # Graph probe -- confirms Plan 03's GET /graph works too (the D-10194        # gate covers reset + graph, not just the search/drift/forget loop).195        # Not part of the timed loop budget.196        graph_data, t6 = _timed_step("6: confirm graph (not in budget)", step_confirm_graph)197        print(f"  graph nodes={len(graph_data['nodes'])} links={len(graph_data.get('links', []))}")198 199        if loop_total >= BUDGET_SECONDS:200            print(f"FAIL: loop took {loop_total:.1f}s, exceeding the {BUDGET_SECONDS:.0f}s budget")201            return 1202 203        if before.get("root_cause") == after.get("root_cause"):204            print("FAIL: root_cause did not change after the release upload + forget")205            return 1206 207        print(f"PASS: loop finished in {loop_total:.1f}s (< {BUDGET_SECONDS:.0f}s budget)")208        return 0209 210    except LoopFailure as exc:211        print(f"FAIL at step: {exc}")212        return 1213    except requests.RequestException as exc:214        print(f"FAIL: HTTP error talking to the backend at {BASE_URL}: {exc}")215        return 1216    finally:217        # Always leave memory in the canonical demo-ready state, regardless218        # of pass/fail (T-04-09 -- never leave memory in a forgotten/mutated219        # state).220        try:221            step_reset()222            print("cleanup: reset -> canonical demo snapshot restored")223        except Exception as exc:  # noqa: BLE001 - best-effort cleanup only224            print(f"WARNING: cleanup reset failed: {exc}")225 226 227if __name__ == "__main__":228    sys.exit(main())229