Benjamin-eecs/openrsi-commit-runtime-assets

1"""Create K sandbox copies per repository for parallel Judge grading.2 3Each copy is `git clone --shared` of the mirror (no object duplication) with4its own `.venv` installed from the repository's requirements.lock, plus the5build backends needed for offline editable rebuilds. Layout:6    <pool_root>/<owner__name>/<i>/   i in 0..K-17 8Usage:9    python -m r2e_local.make_pool --mirrors <repos root> --bootstrap <bootstrap root> \10        --pool <pool root> --copies 2 [--repos owner/name ...] [--python-root <uv python dir>]11"""12 13from __future__ import annotations14 15import argparse16import json17import os18import subprocess19import sys20from concurrent.futures import ThreadPoolExecutor21from pathlib import Path22 23BACKENDS = [24    "setuptools", "wheel", "setuptools-scm", "hatchling", "hatch-vcs", "hatch-fancy-pypi-readme",25    "poetry-core", "flit-core", "pdm-backend", "versioningit", "pbr", "editables",26    "hatch-requirements-txt", "setuptools-git-versioning", "poetry-dynamic-versioning",27]28 29 30def make_one(mirror: Path, boot: dict, lock: Path, dest: Path, env: dict) -> str:31    if (dest / ".venv" / "bin" / "python").exists():32        return f"exists {dest}"33    dest.parent.mkdir(parents=True, exist_ok=True)34    if not (dest / ".git").exists():35        subprocess.run(["git", "clone", "-q", "--shared", "--no-checkout", f"file://{mirror}", str(dest)], check=True)36    subprocess.run(["git", "-C", str(dest), "checkout", "-q", "--detach", boot["ref"]], check=True)37    py = boot.get("extra", {}).get("python_version", "3.12")38    py = ".".join(py.split(".")[:2])39    subprocess.run(["uv", "venv", "-q", "--python", py, ".venv"], cwd=dest, env=env, check=True)40    # The lock is a `uv pip freeze` of the bootstrap venv: drop the editable41    # self-reference (`-e file:///.../sandboxes/<repo>`) and any other local42    # path pins; the copy gets its own editable install via rebuild_cmds below.43    pinned = [44        line for line in lock.read_text().splitlines()45        if line.strip() and not line.startswith(("-e ", "#")) and "file://" not in line46    ]47    lock_copy = dest / ".venv" / "requirements.pool.txt"48    lock_copy.write_text("\n".join(pinned) + "\n")49    subprocess.run(["uv", "pip", "install", "-q", "--python", ".venv/bin/python", "-r", str(lock_copy)], cwd=dest, env=env, check=True)50    subprocess.run(["uv", "pip", "install", "-q", "--python", ".venv/bin/python", *BACKENDS], cwd=dest, env=env, check=False)51    for cmd in boot.get("rebuild_cmds") or []:52        subprocess.run(["bash", "-c", cmd], cwd=dest, env=env, check=False)53    return f"built {dest}"54 55 56def main(argv: list[str] | None = None) -> int:57    ap = argparse.ArgumentParser()58    ap.add_argument("--mirrors", type=Path, required=True)59    ap.add_argument("--bootstrap", type=Path, required=True)60    ap.add_argument("--pool", type=Path, required=True)61    ap.add_argument("--copies", type=int, default=2)62    ap.add_argument("--repos", nargs="*", default=None)63    ap.add_argument("--workers", type=int, default=4)64    args = ap.parse_args(argv)65    env = dict(os.environ)66    env.setdefault("UV_LINK_MODE", "hardlink")67    jobs = []68    for bdir in sorted(args.bootstrap.iterdir()):69        bj = bdir / "bootstrap.json"70        if not bj.exists():71            continue72        boot = json.loads(bj.read_text())73        if args.repos and boot["repo"] not in args.repos:74            continue75        if not boot.get("smoke_passed"):76            continue77        key = bdir.name78        for i in range(args.copies):79            jobs.append((args.mirrors / key, boot, bdir / "requirements.lock", args.pool / key / str(i)))80    with ThreadPoolExecutor(max_workers=args.workers) as ex:81        for line in ex.map(lambda j: make_one(*j, env), jobs):82            print(line, flush=True)83    return 084 85 86if __name__ == "__main__":87    sys.exit(main())88