PRANAV05092003/autonomous-code-refactoring-env
0
1"""2ACRE pre-submission validator.3 4Checks the repository against the submission checklist and, when a server URL is5available, probes the HTTP API as well.6 7Run:8 python validate.py --url http://localhost:78609"""10from __future__ import annotations11 12import argparse13import ast14import re15import sys16from typing import Any, Tuple17 18try:19 import requests20except ImportError:21 print("[ERROR] requests is required. Run: pip install requests")22 sys.exit(1)23 24PASS = "\033[92m[PASS]\033[0m"25FAIL = "\033[91m[FAIL]\033[0m"26 27 28def check(label: str, ok: bool, detail: str = "") -> bool:29 status = PASS if ok else FAIL30 message = f" {status} {label}"31 if detail:32 message += f" - {detail}"33 print(message)34 return ok35 36 37def get(url: str, path: str, timeout: int = 15) -> Tuple[bool, Any]:38 try:39 response = requests.get(f"{url}{path}", timeout=timeout)40 response.raise_for_status()41 return True, response.json()42 except Exception as exc:43 return False, str(exc)44 45 46def post(url: str, path: str, payload: dict, timeout: int = 15) -> Tuple[bool, Any]:47 try:48 response = requests.post(f"{url}{path}", json=payload, timeout=timeout)49 response.raise_for_status()50 return True, response.json()51 except Exception as exc:52 return False, str(exc)53 54 55def read_text(path: str) -> str:56 with open(path, encoding="utf-8") as handle:57 return handle.read()58 59 60def run_validation(base_url: str) -> int:61 failures = 062 63 print("\n" + "=" * 60)64 print(" ACRE Pre-Submission Validator")65 print("=" * 60)66 print(f" Target: {base_url}\n")67 68 print("1. Static repository checks")69 try:70 interface_src = read_text("openenv_interface.py")71 tree = ast.parse(interface_src)72 classes = {node.name: node for node in tree.body if isinstance(node, ast.ClassDef)}73 env_cls = classes.get("OpenEnvRefactorEnv")74 failures += 0 if check("openenv_interface.py exists", True) else 175 failures += 0 if check("OpenEnvRefactorEnv is defined", env_cls is not None) else 176 if env_cls is not None:77 methods = {node.name for node in env_cls.body if isinstance(node, ast.FunctionDef)}78 for method_name in ["reset", "step", "state"]:79 failures += 0 if check(80 f"OpenEnvRefactorEnv implements {method_name}()",81 method_name in methods,82 ) else 183 except FileNotFoundError:84 failures += 185 check("openenv_interface.py exists", False, "file not found")86 87 try:88 models_src = read_text("models.py")89 for name in ["ObservationModel", "ActionModel", "RewardModel"]:90 failures += 0 if check(91 f"{name} is defined in models.py",92 f"class {name}" in models_src,93 ) else 194 except FileNotFoundError:95 failures += 196 check("models.py exists", False, "file not found")97 98 print("\n2. Health check (GET /)")99 ok, data = get(base_url, "/")100 failures += 0 if check("GET / returns HTTP 200", ok) else 1101 if ok:102 failures += 0 if check(103 "Response has status field",104 isinstance(data, dict) and "status" in data,105 str(data),106 ) else 1107 108 print("\n3. Tasks (GET /tasks)")109 ok, data = get(base_url, "/tasks")110 failures += 0 if check("GET /tasks returns 200", ok) else 1111 if ok:112 tasks = data.get("tasks", []) if isinstance(data, dict) else []113 failures += 0 if check("At least 3 tasks defined", len(tasks) >= 3, f"found {len(tasks)}") else 1114 difficulties = [t.get("difficulty", "") for t in tasks]115 for diff in ["easy", "medium", "hard"]:116 failures += 0 if check(f"Task with difficulty '{diff}' exists", diff in difficulties) else 1117 for task in tasks:118 failures += 0 if check(119 f"Task '{task.get('id')}' has initial_code",120 bool(task.get("initial_code")),121 ) else 1122 123 print("\n4. Reset (POST /reset)")124 ok, data = post(base_url, "/reset", {})125 failures += 0 if check("POST /reset returns 200", ok) else 1126 if ok:127 observation = data.get("observation", {})128 failures += 0 if check("Response has observation field", isinstance(observation, dict)) else 1129 failures += 0 if check(130 "Observation is typed with 4 fields",131 {"code_length", "complexity_score", "runtime_s", "error_flag"}.issubset(observation),132 str(observation),133 ) else 1134 135 ok, _ = post(base_url, "/reset", {"task_id": "rename_variables"})136 failures += 0 if check("POST /reset with task_id works", ok) else 1137 138 print("\n5. State (GET /state)")139 ok, data = get(base_url, "/state")140 failures += 0 if check("GET /state returns 200", ok) else 1141 if ok:142 required_keys = [143 "current_code",144 "episode_steps",145 "max_steps",146 "complexity",147 "observation",148 "observation_vector",149 "action_meanings",150 ]151 for key in required_keys:152 failures += 0 if check(f"State has '{key}' field", key in data) else 1153 154 print("\n6. Step (POST /step)")155 post(base_url, "/reset", {"task_id": "rename_variables"})156 for action in range(5):157 ok, data = post(base_url, "/step", {"action": action})158 failures += 0 if check(159 f"Action {action} executes without error",160 ok and isinstance(data, dict) and "reward" in data and "done" in data,161 ) else 1162 if ok:163 reward_payload = data.get("reward", {})164 norm = reward_payload.get("normalized", -1)165 failures += 0 if check(166 f"Action {action} returns typed reward payload",167 {"raw", "normalized", "components"}.issubset(reward_payload),168 str(reward_payload),169 ) else 1170 failures += 0 if check(171 f"Action {action} normalized_reward in [0,1]",172 isinstance(norm, (int, float)) and 0.0 <= float(norm) <= 1.0,173 f"got {norm}",174 ) else 1175 if data.get("done"):176 break177 178 ok, data = post(base_url, "/step", {"action": 99})179 check("Invalid action returns error (not crash)", not ok or "detail" in str(data), "(expected 4xx)")180 181 print("\n7. Task graders (POST /tasks/{id}/grade)")182 for task_id in ["rename_variables", "remove_dead_code", "full_refactor"]:183 ok, data = post(base_url, f"/tasks/{task_id}/grade", {"code": "def f(): pass"})184 failures += 0 if check(f"Grade endpoint for '{task_id}' works", ok) else 1185 if ok:186 score = data.get("score", -1)187 failures += 0 if check(188 f"Score for '{task_id}' in [0.0, 1.0]",189 isinstance(score, (int, float)) and 0.0 <= float(score) <= 1.0,190 f"got {score}",191 ) else 1192 193 print("\n8. openenv.yaml")194 try:195 openenv_yaml = read_text("openenv.yaml")196 failures += 0 if check("openenv.yaml exists", True) else 1197 for field in ["tasks:", "action_space:", "observation_space:", "reward:", "entrypoint:", "validation:"]:198 failures += 0 if check(f"openenv.yaml has '{field}' section", field in openenv_yaml) else 1199 except FileNotFoundError:200 failures += 1201 check("openenv.yaml exists", False, "file not found")202 203 print("\n9. inference.py")204 try:205 inference_src = read_text("inference.py")206 failures += 0 if check("inference.py exists", True) else 1207 # Accept legacy JSON markers and modern strict bracketed format:208 # [START] task=<task_id>209 # [STEP] action=<action>210 # [END] task=<task_id> score=<score>211 json_markers_ok = all(m in inference_src for m in ['"event": "START"', '"event": "STEP"', '"event": "END"'])212 bracket_markers_ok = all(m in inference_src for m in ["[START]", "[STEP]", "[END]"])213 line_markers_ok = all(m in inference_src for m in ["START ", "STEP ", "END "])214 failures += 0 if check("inference.py emits START marker", json_markers_ok or line_markers_ok or bracket_markers_ok) else 1215 failures += 0 if check("inference.py emits STEP marker", json_markers_ok or line_markers_ok or bracket_markers_ok) else 1216 failures += 0 if check("inference.py emits END marker", json_markers_ok or line_markers_ok or bracket_markers_ok) else 1217 failures += 0 if check(218 "Uses OpenAI client",219 "from openai import OpenAI" in inference_src,220 ) else 1221 for var in ["API_BASE_URL", "MODEL_NAME", "ENV_URL", "LOCAL_IMAGE_NAME"]:222 failures += 0 if check(f"inference.py reads {var} from env", var in inference_src) else 1223 failures += 0 if check(224 "inference.py reads API credentials from env (API_KEY or HF_TOKEN)",225 ("API_KEY" in inference_src) or ("HF_TOKEN" in inference_src),226 ) else 1227 api_base_default_ok = (228 'os.getenv("API_BASE_URL", "https://api.openai.com/v1")' in inference_src229 or re.search(r'API_BASE_URL\s*=.*os\.getenv\("API_BASE_URL"\)\s*or\s*"https://api\.openai\.com/v1"', inference_src)230 is not None231 )232 api_base_env_required_ok = (233 re.search(r'base_url\s*=\s*os\.getenv\("API_BASE_URL"\)', inference_src) is not None234 or re.search(r'base_url\s*=\s*os\.environ\["API_BASE_URL"\]', inference_src) is not None235 )236 failures += 0 if check(237 "API_BASE_URL handling is valid (default or strict env)",238 api_base_default_ok or api_base_env_required_ok,239 ) else 1240 241 model_default_ok = (242 'os.getenv("MODEL_NAME", "gpt-4o-mini")' in inference_src243 or re.search(r'MODEL_NAME\s*=.*os\.getenv\("MODEL_NAME"\)\s*or\s*"gpt-4o-mini"', inference_src) is not None244 )245 failures += 0 if check("MODEL_NAME has a default", model_default_ok) else 1246 247 api_key_no_default_ok = (248 re.search(r'API_KEY\s*=.*os\.getenv\("API_KEY"\)', inference_src) is not None249 and re.search(r'os\.getenv\("API_KEY"\s*,', inference_src) is None250 )251 hf_token_no_default_ok = (252 re.search(r'HF_TOKEN\s*=.*os\.getenv\("HF_TOKEN"\)', inference_src) is not None253 and re.search(r'os\.getenv\("HF_TOKEN"\s*,', inference_src) is None254 )255 failures += 0 if check(256 "API key variable has no default",257 api_key_no_default_ok or hf_token_no_default_ok,258 ) else 1259 except FileNotFoundError:260 failures += 1261 check("inference.py exists", False, "file not found")262 263 print("\n10. Dockerfile")264 try:265 dockerfile = read_text("Dockerfile")266 failures += 0 if check("Dockerfile exists", True) else 1267 failures += 0 if check("Exposes port 7860", "7860" in dockerfile) else 1268 failures += 0 if check("Has CMD/ENTRYPOINT", "CMD" in dockerfile or "ENTRYPOINT" in dockerfile) else 1269 failures += 0 if check("Does not set a default HF_TOKEN", "ENV HF_TOKEN" not in dockerfile) else 1270 except FileNotFoundError:271 failures += 1272 check("Dockerfile exists", False, "file not found")273 274 print("\n11. README / Hugging Face metadata")275 try:276 readme = read_text("README.md")277 failures += 0 if check("README has docker SDK front matter", "sdk: docker" in readme) else 1278 failures += 0 if check("README includes openenv tag", "openenv" in readme) else 1279 for section in [280 "Environment Overview and Motivation",281 "Definitions of Action and Observation Spaces",282 "Task Descriptions with Expected Difficulty Levels",283 "Setup and Usage Instructions",284 "Baseline Performance Scores",285 ]:286 failures += 0 if check(f"README includes '{section}'", section in readme) else 1287 except FileNotFoundError:288 failures += 1289 check("README.md exists", False, "file not found")290 291 print("\n" + "=" * 60)292 if failures == 0:293 print(f" {PASS} All checks passed. Repository is submission-ready.")294 else:295 print(f" {FAIL} {failures} check(s) failed. Fix before submitting.")296 print("=" * 60 + "\n")297 298 return failures299 300 301def main() -> None:302 parser = argparse.ArgumentParser(description="ACRE pre-submission validator")303 parser.add_argument(304 "--url",305 default="http://localhost:7860",306 help="Base URL of the running ACRE server",307 )308 args = parser.parse_args()309 sys.exit(run_validation(args.url))310 311 312if __name__ == "__main__":313 main()314