Benjamin-eecs/openrsi-commit-runtime-assets

1#!/usr/bin/env python32"""Prepare one corpus repository for LocalSandbox validation and write bootstrap.json.3 4Replaces Repo2RLEnv's Docker-based `ensure_bootstrap` (bootstrap/runner.py line 4715requires a daemon). Produces the same BootstrapResult shape that6bootstrap/cache.py line 117 loads.7 8Steps9  1. sandbox checkout: `git clone --shared` of the local mirror, detached at the pinned SHA10  2. `.venv` inside the checkout (excluded by the stage script's `git clean -e .venv`)11  3. install: editable package + test extras + requirements files + pytest12  4. verify: pytest collection, then a bounded full run13  5. write bootstrap.json, requirements.lock, report.json14 15Usage:16  bootstrap_repo.py --repo owner/name --sha <pinned> [--python 3.12] [--extra-pkg X ...]17                    [--install-cmd "<shell>"] [--test-cmd "python -m pytest"] [--suite-timeout 1200]18Exit code 0 even when verification fails; read report.json / the printed summary.19"""20 21from __future__ import annotations22 23import argparse24import configparser25import json26import os27import re28import shutil29import subprocess30import sys31import time32import tomllib33from pathlib import Path34 35A = Path("/Users/beneecs/work/projects/openrsi/code/assets")36REQ_FILE_CANDIDATES = [37    "requirements-dev.txt",38    "requirements_dev.txt",39    "dev-requirements.txt",40    "requirements-test.txt",41    "requirements_test.txt",42    "test-requirements.txt",43    "test_requirements.txt",44    "requirements/dev.txt",45    "requirements/test.txt",46    "requirements/tests.txt",47    "requirements/testing.txt",48    "tests/requirements.txt",49]50EXTRA_KEYS = ("test", "tests", "testing", "dev", "develop", "development")51 52 53def sh(cmd: str, *, cwd: Path, env: dict, timeout: int) -> tuple[int, str, float]:54    t = time.monotonic()55    try:56        p = subprocess.run(57            ["bash", "-lc", cmd], cwd=str(cwd), env=env, capture_output=True, text=True,58            errors="replace", timeout=timeout, check=False,59        )60        return p.returncode, (p.stdout + "\n--- stderr ---\n" + p.stderr), time.monotonic() - t61    except subprocess.TimeoutExpired as exc:62        out = exc.stdout.decode(errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")63        return 124, out + f"\n[timeout after {timeout}s]", time.monotonic() - t64 65 66def requires_python(repo: Path) -> str | None:67    py = repo / "pyproject.toml"68    if py.exists():69        try:70            d = tomllib.loads(py.read_text())71            rp = d.get("project", {}).get("requires-python")72            if rp:73                return rp74            poetry = d.get("tool", {}).get("poetry", {}).get("dependencies", {}).get("python")75            if poetry:76                return str(poetry)77        except Exception:78            pass79    cfg = repo / "setup.cfg"80    if cfg.exists():81        cp = configparser.ConfigParser()82        try:83            cp.read(cfg)84            if cp.has_option("options", "python_requires"):85                return cp.get("options", "python_requires")86        except Exception:87            pass88    sp = repo / "setup.py"89    if sp.exists():90        m = re.search(r"python_requires\s*=\s*['\"]([^'\"]+)['\"]", sp.read_text(errors="replace"))91        if m:92            return m.group(1)93    return None94 95 96def choose_python(spec: str | None, preferred: str) -> str:97    if not spec:98        return preferred99    try:100        from packaging.specifiers import SpecifierSet101        from packaging.version import Version102    except ImportError:103        return preferred104    try:105        ss = SpecifierSet(spec.replace("^", ">="))106    except Exception:107        return preferred108    for cand in [preferred, "3.12", "3.11", "3.10", "3.9"]:109        if ss.contains(Version(cand + ".0"), prereleases=True) or ss.contains(Version(cand + ".99")):110            return cand111    return preferred112 113 114def discover_extras(repo: Path) -> tuple[list[str], list[str], list[str]]:115    """Return (extras, requirement files, poetry dev packages)."""116    extras: list[str] = []117    poetry_dev: list[str] = []118    py = repo / "pyproject.toml"119    if py.exists():120        try:121            d = tomllib.loads(py.read_text())122            opt = d.get("project", {}).get("optional-dependencies", {})123            extras += [k for k in opt if k.lower() in EXTRA_KEYS]124            groups = d.get("tool", {}).get("poetry", {}).get("group", {})125            for g, body in groups.items():126                if g.lower() in EXTRA_KEYS:127                    poetry_dev += list(body.get("dependencies", {}).keys())128            legacy = d.get("tool", {}).get("poetry", {}).get("dev-dependencies", {})129            poetry_dev += list(legacy.keys())130            pdm = d.get("tool", {}).get("pdm", {}).get("dev-dependencies", {})131            for g, deps in pdm.items():132                if g.lower() in EXTRA_KEYS:133                    poetry_dev += [re.split(r"[<>=!~\[ ;]", x)[0] for x in deps]134            dg = d.get("dependency-groups", {})135            for g, deps in dg.items():136                if g.lower() in EXTRA_KEYS:137                    poetry_dev += [re.split(r"[<>=!~\[ ;]", x)[0] for x in deps if isinstance(x, str)]138        except Exception:139            pass140    cfg = repo / "setup.cfg"141    if cfg.exists():142        cp = configparser.ConfigParser()143        try:144            cp.read(cfg)145            if cp.has_section("options.extras_require"):146                extras += [k for k in cp.options("options.extras_require") if k.lower() in EXTRA_KEYS]147        except Exception:148            pass149    reqs = [r for r in REQ_FILE_CANDIDATES if (repo / r).exists()]150    poetry_dev = [p for p in dict.fromkeys(poetry_dev) if p.lower() != "python"]151    return list(dict.fromkeys(extras)), reqs, poetry_dev152 153 154def main() -> int:155    ap = argparse.ArgumentParser()156    ap.add_argument("--repo", required=True)157    ap.add_argument("--sha", required=True)158    ap.add_argument("--python", default="3.12")159    ap.add_argument("--extra-pkg", action="append", default=[])160    ap.add_argument("--install-cmd", action="append", default=[], help="extra shell after base install")161    ap.add_argument("--test-cmd", default="python -m pytest")162    ap.add_argument("--suite-timeout", type=int, default=1200)163    ap.add_argument("--skip-suite", action="store_true")164    ap.add_argument("--force", action="store_true", help="recreate sandbox and venv")165    args = ap.parse_args()166 167    key = args.repo.replace("/", "__")168    mirror = A / "repos" / key169    sandbox = A / "sandboxes" / key170    home = A / "sandboxes" / f".home-{key}"171    bdir = A / "bootstrap" / key172    bdir.mkdir(parents=True, exist_ok=True)173    home.mkdir(parents=True, exist_ok=True)174    log: list[dict] = []175    t_start = time.monotonic()176 177    if not (mirror / ".git").exists():178        print(json.dumps({"repo": args.repo, "status": "no_mirror"}))179        return 0180 181    if args.force and sandbox.exists():182        shutil.rmtree(sandbox)183    if not (sandbox / ".git").exists():184        sandbox.parent.mkdir(parents=True, exist_ok=True)185        subprocess.run(["git", "clone", "-q", "--shared", "--no-checkout", f"file://{mirror}", str(sandbox)], check=True)186    subprocess.run(["git", "-C", str(sandbox), "checkout", "-q", "--detach", args.sha], check=True)187    subprocess.run(["git", "-C", str(sandbox), "clean", "-fdxq", "-e", ".venv"], check=False)188 189    rp = requires_python(sandbox)190    pyver = choose_python(rp, args.python)191    venv = sandbox / ".venv"192    env = {193        "PATH": f"{venv}/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",194        "HOME": str(home), "VIRTUAL_ENV": str(venv), "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8",195        "PYTHONDONTWRITEBYTECODE": "1", "PIP_DISABLE_PIP_VERSION_CHECK": "1",196        "UV_CACHE_DIR": str(A / ".uv-cache"), "UV_PYTHON_INSTALL_DIR": str(A / ".uv-python"),197        "GIT_TERMINAL_PROMPT": "0",198    }199 200    def step(name: str, cmd: str, timeout: int = 900) -> int:201        rc, out, dt = sh(cmd, cwd=sandbox, env=env, timeout=timeout)202        log.append({"step": name, "cmd": cmd, "rc": rc, "sec": round(dt, 1), "tail": out[-3000:]})203        return rc204 205    if args.force or not (venv / "bin" / "python").exists():206        if venv.exists():207            shutil.rmtree(venv)208        step("venv", f"uv venv -q --python {pyver} .venv", 300)209 210    extras, reqs, poetry_dev = discover_extras(sandbox)211    install_cmds: list[str] = []212    pip = "uv pip install -q --python .venv/bin/python"213    base = f"{pip} -e ."214    if extras:215        base = f"{pip} -e '.[{','.join(extras)}]'"216    install_cmds.append(base)217    for r in reqs:218        install_cmds.append(f"{pip} -r {r}")219    if poetry_dev:220        install_cmds.append(f"{pip} {' '.join(repr(p) for p in poetry_dev)}")221    install_cmds.append(f"{pip} pytest")222    for extra in args.extra_pkg:223        install_cmds.append(f"{pip} {extra}")224    install_cmds += args.install_cmd225 226    install_ok = True227    for i, cmd in enumerate(install_cmds):228        rc = step(f"install_{i}", cmd, 900)229        if rc != 0 and cmd == base:230            # Fall back to a plain editable install if extras are broken.231            rc = step(f"install_{i}_fallback", f"{pip} -e .", 900)232        if rc != 0 and cmd in (base, f"{pip} pytest"):233            install_ok = False234 235    collect_ok = step("collect", "python -m pytest -q --co -p no:cacheprovider 2>&1 | tail -5", 600) == 0236    collected = None237    if log[-1]["tail"]:238        m = re.search(r"(\d+) tests? collected", log[-1]["tail"]) or re.search(r"(\d+)/(\d+) tests collected", log[-1]["tail"])239        if m:240            collected = int(m.group(1))241 242    suite = {"ran": False}243    if not args.skip_suite:244        rc = step("suite", f"{args.test_cmd} -q -p no:cacheprovider -x --maxfail=50 2>&1 | tail -15", args.suite_timeout)245        tail = log[-1]["tail"]246        passed = re.search(r"(\d+) passed", tail)247        failed = re.search(r"(\d+) failed", tail)248        errors = re.search(r"(\d+) errors?", tail)249        suite = {250            "ran": True, "rc": rc, "sec": log[-1]["sec"],251            "passed": int(passed.group(1)) if passed else 0,252            "failed": int(failed.group(1)) if failed else 0,253            "errors": int(errors.group(1)) if errors else 0,254        }255 256    step("freeze", "uv pip freeze --python .venv/bin/python > ../../bootstrap/" + key + "/requirements.lock", 120)257    step("clean", "git reset -q --hard HEAD && git clean -fdxq -e .venv", 120)258 259    pyv = subprocess.run([str(venv / "bin" / "python"), "-c", "import sys;print('.'.join(map(str,sys.version_info[:3])))"], capture_output=True, text=True).stdout.strip()260    smoke = install_ok and collect_ok and (not suite.get("ran") or suite.get("passed", 0) > 0)261    bootstrap = {262        "image_digest": f"local-venv:{key}",263        "image_tag": f"local/{key}",264        "language": "python",265        "repo": args.repo,266        "ref": args.sha,267        "rebuild_cmds": ["python -m pip install --no-deps --no-build-isolation -e ."],268        "test_cmds": [args.test_cmd],269        "smoke_passed": bool(smoke),270        "iterations": 0,271        "build_time_sec": round(time.monotonic() - t_start, 1),272        "llm_provider": "none",273        "llm_cost_estimate_usd": 0.0,274        "dockerfile_reconstruction": "",275        "transcript_path": None,276        "pushed_to_registry": False,277        "verify_passed": bool(smoke),278        "verify_detail": f"python {pyv}; collected={collected}; suite={suite}",279        "extra": {280            "python_version": pyv,281            "requires_python": rp,282            "install_cmds": install_cmds,283            "extras": extras,284            "requirement_files": reqs,285            "poetry_dev": poetry_dev,286            "collected": collected,287            "suite": suite,288            "sandbox_layout": "checkout with .venv inside; LocalSandbox adapter",289        },290    }291    (bdir / "bootstrap.json").write_text(json.dumps(bootstrap, indent=2))292    (bdir / "report.json").write_text(json.dumps({"log": log}, indent=2))293    print(json.dumps({294        "repo": args.repo, "python": pyv, "install_ok": install_ok, "collect_ok": collect_ok,295        "collected": collected, "suite": suite, "smoke_passed": smoke,296        "extras": extras, "reqs": reqs, "poetry_dev": poetry_dev[:8],297    }))298    return 0299 300 301if __name__ == "__main__":302    sys.exit(main())303