jester1177/cloudnative-devops-debug-env
0
1"""2Inference Script for Cloud-Native Debug Environment3===================================4MANDATORY5- Before submitting, ensure the following variables are defined in your environment configuration:6 API_BASE_URL The API endpoint for the LLM.7 MODEL_NAME The model identifier to use for inference.8 HF_TOKEN Your Hugging Face / API key.9 LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()10 method11 12- Defaults are set only for API_BASE_URL and MODEL_NAME13 (and should reflect your active inference setup):14 API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")15 MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")16 17- The inference script must be named `inference.py` and placed in the root directory of the project18- Participants must use OpenAI Client for all LLM calls using above variables19 20STDOUT FORMAT21- The script must emit exactly three line types to stdout, in this order:22 23 [START] task=<task_name> env=<benchmark> model=<model_name>24 [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>25 [END] success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>26 27 Rules:28 - One [START] line at episode begin.29 - One [STEP] line per step, immediately after env.step() returns.30 - One [END] line after the episode completes, always emitted (even on exception).31 - reward and rewards are formatted to 2 decimal places.32 - done and success are lowercase booleans: true or false.33 - error is the raw error string, or null if none.34 - All fields on a single line with no newlines within a line.35 - Each tasks should return score in [0, 1]36 37 Example:38 [START] task=dockerfile_syntax env=cloud_native_devops model=meta-llama/Llama-3.1-70B-Instruct39 [STEP] step=1 action=edit_file reward=0.30 done=false error=null40 [STEP] step=2 action=submit reward=0.00 done=true error=null41 [END] success=true steps=2 score=0.850 rewards=0.30,0.0042"""43 44 45import json46import os47import re48import sys49import time50from typing import Any, Dict, List, Optional51 52import requests53from openai import OpenAI54 55 56API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"57MODEL_NAME = os.getenv("MODEL_NAME") or "meta-llama/Llama-3.1-70B-Instruct"58API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")59ENV_URL = os.getenv("ENV_URL", "http://localhost:7860")60LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")61BENCHMARK = "cloud_native_devops"62MAX_STEPS = 8 # leave 2 steps buffer before env hard-limit of 1063SUCCESS_SCORE_THRESHOLD = 0.1 # normalized score in [0, 1]64 65SYSTEM_PROMPT = """You are an expert DevOps engineer debugging cloud-native deployment pipelines.66You will receive broken Dockerfile, GitHub Actions workflow, and/or Kubernetes manifest files along with error messages.67 68Your job is to:691. Analyze the error message carefully702. Identify the root cause in the configuration files713. Provide a precise fix72 73When you identify a fix, respond with a JSON object in this exact format:74{75 "action_type": "YOUR_CHOSEN_ACTION_TYPE",76 "reasoning": "Brief explanation of the bug and fix",77 "edits": [78 {79 "file_path": "path/to/file",80 "line_number": 5, // Only needed for replace_line, add_line, delete_line, add_block81 "old_content": "exactly broken", // Only needed for edit_file, delete_block82 "new_content": "corrected block" // Not needed for delete_line, delete_block83 }84 ]85}86 87Available action_type values for edits:88- "edit_file" (requires old_content and new_content)89- "replace_line" (requires line_number and new_content)90- "add_line" (requires line_number and new_content)91- "delete_line" (requires line_number)92- "add_block" (requires line_number and new_content)93- "delete_block" (requires old_content)94 95To create a new file (e.g. a missing ConfigMap), use "edit_file" with empty old_content:96{97 "action_type": "edit_file",98 "reasoning": "Create missing ConfigMap manifest",99 "edits": [100 {101 "file_path": "k8s/configmap.yaml",102 "old_content": "",103 "new_content": "apiVersion: v1\\nkind: ConfigMap\\n..."104 }105 ]106}107 108If you believe all issues are fixed and want to submit, respond with:109{"action_type": "submit"}110 111If you need a hint, respond with:112{"action_type": "request_hint"}113 114Rules:115- Match old_content EXACTLY as it appears in the file (whitespace matters)116- Fix one issue at a time for precision117- Focus on the error message — it tells you exactly what's wrong118- Common issues: typos, wrong syntax, missing fields, wrong secret references119- For GitHub Actions: check secret syntax (${{ }} not ${ }), env blocks, permissions120- For Dockerfiles: check instruction syntax, file paths, base image tags121- For Kubernetes: check label selectors, port matching, resource limits, probe configs, ingress rules122- For full-stack pipelines: issues may span multiple files (workflow + Dockerfile + K8s manifests)123- Always respond with valid JSON only, no markdown fences"""124 125 126# ---------------------------------------------------------------------------127# Logging helpers (mandatory stdout format)128# ---------------------------------------------------------------------------129 130def log_start(task: str, env: str, model: str) -> None:131 print(f"[START] task={task} env={env} model={model}", flush=True)132 133 134def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:135 error_val = error if error else "null"136 done_val = str(done).lower()137 print(138 f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",139 flush=True,140 )141 142 143def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:144 rewards_str = ",".join(f"{r:.2f}" for r in rewards)145 print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)146 147 148# ---------------------------------------------------------------------------149# Client / env helpers150# ---------------------------------------------------------------------------151 152def create_client() -> OpenAI:153 """Create OpenAI-compatible client for HuggingFace router."""154 return OpenAI(155 base_url=API_BASE_URL,156 api_key=API_KEY,157 )158 159 160def env_request(method: str, endpoint: str, json_data: Optional[Dict] = None) -> Dict[str, Any]:161 """Make a request to the environment server."""162 url = f"{ENV_URL}{endpoint}"163 if method == "GET":164 resp = requests.get(url, timeout=30)165 else:166 resp = requests.post(url, json=json_data or {}, timeout=30)167 resp.raise_for_status()168 return resp.json()169 170 171def format_observation(obs: Dict[str, Any]) -> str:172 """Format observation into a prompt for the LLM."""173 parts = []174 parts.append(f"Task: {obs.get('task_description', 'Unknown')}")175 parts.append(f"Difficulty: {obs.get('difficulty', 'unknown')}")176 parts.append(f"Step: {obs.get('step_number', 0)}/{obs.get('max_steps', 10)}")177 parts.append(f"Issues fixed: {obs.get('issues_fixed', 0)}/{obs.get('total_issues', '?')}")178 179 error = obs.get("error", {})180 parts.append(f"\n--- ERROR ---")181 parts.append(f"Phase: {error.get('phase', 'unknown')}")182 parts.append(f"Message: {error.get('error_message', 'No error')}")183 if error.get("failed_step"):184 parts.append(f"Failed step: {error['failed_step']}")185 if error.get("line_hint"):186 parts.append(f"Line hint: {error['line_hint']}")187 188 parts.append(f"\n--- FILES ---")189 for f in obs.get("files", []):190 parts.append(f"\n=== {f['path']} ({f.get('file_type', 'unknown')}) ===")191 content = f.get("content", "")192 lines = content.split("\n")193 for i, line in enumerate(lines, 1):194 parts.append(f"{i:3d} | {line}")195 196 if obs.get("available_secrets"):197 parts.append(f"\n--- AVAILABLE SECRETS ---")198 parts.append(", ".join(obs["available_secrets"]))199 200 if obs.get("last_action_feedback"):201 parts.append(f"\n--- LAST ACTION FEEDBACK ---")202 parts.append(obs["last_action_feedback"])203 204 return "\n".join(parts)205 206 207def parse_llm_response(text: str) -> Dict[str, Any]:208 """Parse LLM response into an action dict."""209 text = text.strip()210 211 # Strip markdown code fences if present212 if text.startswith("```"):213 lines = text.split("\n")214 lines = [l for l in lines if not l.strip().startswith("```")]215 text = "\n".join(lines).strip()216 217 # Try to find JSON in the response218 json_match = re.search(r'\{[\s\S]*\}', text)219 if json_match:220 try:221 return json.loads(json_match.group())222 except json.JSONDecodeError:223 pass224 225 # Fallback: treat as submit226 return {"action": "submit"}227 228 229def build_action(parsed: Dict[str, Any]) -> Dict[str, Any]:230 """Convert parsed LLM response to environment action format."""231 action_type = parsed.get("action_type")232 233 # Backwards compatibility and standard aliases234 if parsed.get("action") == "submit" or action_type == "submit":235 return {"action_type": "submit"}236 if parsed.get("action") == "hint" or action_type == "request_hint":237 return {"action_type": "request_hint"}238 239 edits = parsed.get("edits", [])240 if not edits and not action_type:241 return {"action_type": "submit"}242 243 action_str = action_type if action_type else "edit_file"244 245 return {246 "action_type": action_str,247 "edits": [248 {249 "file_path": e.get("file_path", ""),250 "line_number": e.get("line_number"),251 "old_content": e.get("old_content", ""),252 "new_content": e.get("new_content", ""),253 }254 for e in edits255 ],256 }257 258 259def run_episode(client: OpenAI, task_id: Optional[str] = None, scenario_id: Optional[str] = None) -> Dict[str, Any]:260 """Run a single episode: reset, loop (observe -> LLM -> act), grade."""261 reset_payload: Dict[str, Any] = {}262 if task_id:263 reset_payload["task_id"] = task_id264 if scenario_id:265 reset_payload["scenario_id"] = scenario_id266 267 # Best-effort task name for Start268 target_task = task_id or "random_task"269 log_start(task=target_task, env=BENCHMARK, model=MODEL_NAME)270 271 trajectory = []272 rewards: List[float] = []273 steps_taken = 0274 score = 0.0275 success = False276 277 try:278 reset_resp = env_request("POST", "/reset", reset_payload)279 obs = reset_resp["observation"]280 info = reset_resp.get("info", {})281 282 actual_task_id = info.get("task_id", target_task)283 actual_scenario_id = info.get("scenario_id", scenario_id or "unknown")284 285 messages = [{"role": "system", "content": SYSTEM_PROMPT}]286 for step_num in range(1, MAX_STEPS + 1):287 user_msg = format_observation(obs)288 messages.append({"role": "user", "content": user_msg})289 290 error_msg: Optional[str] = None291 292 try:293 completion = client.chat.completions.create(294 model=MODEL_NAME,295 messages=messages,296 temperature=0.1,297 max_tokens=1024,298 )299 llm_text = completion.choices[0].message.content or '{"action": "submit"}'300 except Exception as e:301 error_msg = str(e)302 print(f"[DEBUG] Model request failed: {e}", flush=True)303 llm_text = '{"action": "submit"}'304 305 messages.append({"role": "assistant", "content": llm_text})306 307 parsed = parse_llm_response(llm_text)308 action = build_action(parsed)309 310 step_resp = env_request("POST", "/step", {"action": action})311 obs = step_resp["observation"]312 reward = step_resp.get("reward", 0.0)313 done = step_resp.get("done", False)314 step_info = step_resp.get("info", {})315 steps_taken = step_num316 317 rewards.append(reward)318 319 log_step(320 step=step_num,321 action=action["action_type"],322 reward=reward,323 done=done,324 error=error_msg,325 )326 327 trajectory.append({328 "step": step_num,329 "action": action,330 "reward": reward,331 "done": done,332 "info": step_info,333 })334 335 if done:336 break337 338 # Grade the trajectory339 grade_resp = env_request("POST", "/grader", {340 "task_id": actual_task_id,341 "trajectory": trajectory,342 })343 result = grade_resp.get("result", {})344 score = result.get("score", 0.0)345 score = min(max(score, 0.0), 1.0) # clamp to [0, 1]346 success = score >= SUCCESS_SCORE_THRESHOLD347 348 finally:349 log_end(success=success, steps=steps_taken, score=score, rewards=rewards)350 351 return {"score": score, "success": success, "steps": steps_taken, "rewards": rewards}352 353 354def run_all_tasks(client: OpenAI) -> Dict[str, float]:355 """Run baseline on all tasks (and ALL their scenarios) and report scores."""356 try:357 from server.tasks.task_registry import TASK_REGISTRY358 except ImportError as e:359 print(f"[DEBUG] Could not import TASK_REGISTRY: {e}", flush=True)360 return {}361 362 scores: Dict[str, List[float]] = {}363 364 for task_id, task_cls in TASK_REGISTRY.items():365 task_scores = []366 367 # Iterate over all exact scenarios for this task368 scenarios = task_cls.SCENARIOS369 for scenario in scenarios:370 scenario_id = scenario["id"]371 result = run_episode(client, task_id=task_id, scenario_id=scenario_id)372 task_scores.append(result.get("score", 0.0))373 374 scores[task_id] = task_scores375 376 # Summary377 print(f"\n[DEBUG] {'='*60}", flush=True)378 print("[DEBUG] BASELINE RESULTS SUMMARY", flush=True)379 print(f"[DEBUG] {'='*60}", flush=True)380 avg_scores = {}381 for task_id, task_scores in scores.items():382 avg = sum(task_scores) / len(task_scores) if task_scores else 0.0383 avg_scores[task_id] = avg384 print(f"[DEBUG] {task_id:40s} {avg:.3f}", flush=True)385 386 overall = sum(avg_scores.values()) / len(avg_scores) if avg_scores else 0.0387 print(f"[DEBUG] {'OVERALL':40s} {overall:.3f}", flush=True)388 389 return avg_scores390 391 392def main():393 """Entry point for baseline inference."""394 if not API_KEY:395 print("[DEBUG] WARNING: HF_TOKEN not set. Set it via: export HF_TOKEN=your_token_here", flush=True)396 print("[DEBUG] Continuing anyway (will fail if auth is required)...", flush=True)397 398 # Verify environment is running399 try:400 health = env_request("GET", "/health")401 print(f"[DEBUG] Environment status: {health.get('status', 'unknown')}", flush=True)402 except Exception as e:403 print(f"[DEBUG] Cannot connect to environment at {ENV_URL}: {e}", flush=True)404 print("[DEBUG] Start the server first: python -m uvicorn server.app:app --host 0.0.0.0 --port 7860", flush=True)405 sys.exit(1)406 407 client = create_client()408 409 # If a specific task is requested via CLI arg410 if len(sys.argv) > 1:411 task_id = sys.argv[1]412 scenario_id = sys.argv[2] if len(sys.argv) > 2 else None413 run_episode(client, task_id=task_id, scenario_id=scenario_id)414 else:415 run_all_tasks(client)416 417 418if __name__ == "__main__":419 main()420