SidhaGarg/Cloud-DevOps-RLEnv

1#!/usr/bin/env bash2#3# pre_submit_validate.sh4#5# Extended pre-submission checks for OpenEnv hackathon submissions.6# This script complements scripts/validate-submission.sh by also checking7# inference contract requirements and baseline reproducibility.8 9set -euo pipefail10 11DOCKER_BUILD_TIMEOUT=60012INFERENCE_TIMEOUT=120013 14PING_URL=""15REPO_DIR="."16SKIP_DOCKER=false17SKIP_INFERENCE=false18PYTHON_BIN=""19OPENENV_BIN=""20OPENENV_USE_MODULE=false21DOCKER_CONTAINER_ID=""22INFERENCE_OUT_FILE=".pre-submit-inference.out"23 24usage() {25  cat <<'EOF'26Usage: scripts/pre_submit_validate.sh [options]27 28Options:29  --ping-url <url>        HF Space URL (e.g., https://team-space.hf.space)30  --repo-dir <path>       Repo root directory (default: current directory)31  --skip-docker           Skip docker build check32  --skip-inference        Skip inference baseline check33  -h, --help              Show this help message34 35Required environment variables for inference checks:36  API_BASE_URL37  MODEL_NAME38  HF_TOKEN39EOF40}41 42run_with_timeout() {43  local secs="$1"; shift44  if command -v timeout >/dev/null 2>&1; then45    timeout "$secs" "$@"46  elif command -v gtimeout >/dev/null 2>&1; then47    gtimeout "$secs" "$@"48  else49    "$@" &50    local pid=$!51    ( sleep "$secs" && kill "$pid" 2>/dev/null ) &52    local watcher=$!53    wait "$pid" 2>/dev/null54    local rc=$?55    kill "$watcher" 2>/dev/null || true56    wait "$watcher" 2>/dev/null || true57    return $rc58  fi59}60 61log() {62  printf "[%s] %s\n" "$(date -u +%H:%M:%S)" "$*"63}64 65die() {66  log "FAILED -- $*"67  exit 168}69 70pass() {71  log "PASSED -- $*"72}73 74cleanup() {75  if [ -n "$DOCKER_CONTAINER_ID" ]; then76    docker rm -f "$DOCKER_CONTAINER_ID" >/dev/null 2>&1 || true77  fi78  rm -f "$INFERENCE_OUT_FILE" >/dev/null 2>&1 || true79}80 81trap cleanup EXIT82 83resolve_python_bin() {84  local candidates=(85    "$REPO_DIR/.venv/bin/python"86    "$REPO_DIR/.venv/Scripts/python.exe"87    "$REPO_DIR/../.venv/bin/python"88    "$REPO_DIR/../.venv/Scripts/python.exe"89  )90 91  for c in "${candidates[@]}"; do92    if [ -x "$c" ]; then93      PYTHON_BIN="$c"94      return 095    fi96  done97 98  if command -v python >/dev/null 2>&1; then99    PYTHON_BIN="$(command -v python)"100    return 0101  fi102  if command -v python3 >/dev/null 2>&1; then103    PYTHON_BIN="$(command -v python3)"104    return 0105  fi106 107  return 1108}109 110resolve_openenv_cmd() {111  local candidates=(112    "$REPO_DIR/.venv/bin/openenv"113    "$REPO_DIR/.venv/Scripts/openenv.exe"114    "$REPO_DIR/../.venv/bin/openenv"115    "$REPO_DIR/../.venv/Scripts/openenv.exe"116  )117 118  for c in "${candidates[@]}"; do119    if [ -x "$c" ]; then120      OPENENV_BIN="$c"121      return 0122    fi123  done124 125  if command -v openenv >/dev/null 2>&1; then126    OPENENV_BIN="$(command -v openenv)"127    return 0128  fi129 130  return 1131}132 133while [ "$#" -gt 0 ]; do134  case "$1" in135    --ping-url)136      shift137      [ "$#" -gt 0 ] || die "--ping-url requires a value"138      PING_URL="$1"139      ;;140    --repo-dir)141      shift142      [ "$#" -gt 0 ] || die "--repo-dir requires a value"143      REPO_DIR="$1"144      ;;145    --skip-docker)146      SKIP_DOCKER=true147      ;;148    --skip-inference)149      SKIP_INFERENCE=true150      ;;151    -h|--help)152      usage153      exit 0154      ;;155    *)156      die "Unknown option: $1"157      ;;158  esac159  shift160done161 162REPO_DIR="$(cd "$REPO_DIR" && pwd)"163cd "$REPO_DIR"164 165log "Repo: $REPO_DIR"166 167resolve_python_bin || die "No usable Python interpreter found"168log "Python: $PYTHON_BIN"169 170if resolve_openenv_cmd; then171  log "OpenEnv CLI: $OPENENV_BIN"172else173  OPENENV_USE_MODULE=true174  log "OpenEnv CLI via module: $PYTHON_BIN -m openenv"175fi176 177log "Step 1/8: Checking OpenEnv standard file layout"178required_files=(179  "openenv.yaml"180  "models.py"181  "env.py"182  "inference.py"183  "server/app.py"184  "server/cloud_devops_env_environment.py"185)186for f in "${required_files[@]}"; do187  [ -f "$f" ] || die "Missing required file: $f"188done189pass "Core OpenEnv file layout looks valid"190 191log "Step 2/8: Checking inference contract requirements"192[ -f "inference.py" ] || die "inference.py must exist in repo root"193grep -q "from openai import OpenAI" inference.py || die "inference.py must import OpenAI client"194grep -q "OpenAI(" inference.py || die "inference.py must instantiate OpenAI client"195grep -q "\[START\]" inference.py || die "inference.py must emit [START] logs"196grep -q "\[STEP\]" inference.py || die "inference.py must emit [STEP] logs"197grep -q "\[END\]" inference.py || die "inference.py must emit [END] logs"198pass "Inference script contract checks passed"199 200log "Step 3/8: Validating OpenEnv manifest and typed models"201if [ "$OPENENV_USE_MODULE" = true ]; then202  "$PYTHON_BIN" -m openenv validate >/tmp/openenv-validate.out 2>&1 || {203    cat /tmp/openenv-validate.out204    die "openenv validate failed"205  }206else207  "$OPENENV_BIN" validate >/tmp/openenv-validate.out 2>&1 || {208    cat /tmp/openenv-validate.out209    die "openenv validate failed"210  }211fi212pass "openenv validate passed"213 214log "Step 4/8: Optional HF Space ping check"215if [ -n "$PING_URL" ]; then216  PING_URL="${PING_URL%/}"217  code=$(curl -s -o /tmp/pre-submit-ping.out -w "%{http_code}" -X POST \218    -H "Content-Type: application/json" -d '{}' \219    "$PING_URL/reset" --max-time 30 || printf "000")220  [ "$code" = "200" ] || die "HF Space /reset returned HTTP $code"221  pass "HF Space responds to /reset (HTTP 200)"222else223  log "SKIPPED -- no --ping-url provided"224fi225 226log "Step 5/8: Docker build + run check"227if [ "$SKIP_DOCKER" = true ]; then228  log "SKIPPED -- --skip-docker enabled"229else230  command -v docker >/dev/null 2>&1 || die "docker not found"231  if [ -f "Dockerfile" ]; then232    context="."233  elif [ -f "server/Dockerfile" ]; then234    context="server"235  else236    die "No Dockerfile found at root or server/"237  fi238  run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$context" >/tmp/pre-submit-docker.out 2>&1 || {239    tail -n 40 /tmp/pre-submit-docker.out240    die "docker build failed"241  }242  pass "Docker build succeeded"243 244  IMAGE_TAG="openenv-pre-submit-local"245  run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build -t "$IMAGE_TAG" "$context" >/tmp/pre-submit-docker-tagged.out 2>&1 || {246    tail -n 40 /tmp/pre-submit-docker-tagged.out247    die "docker build (tagged) failed"248  }249 250  DOCKER_CONTAINER_ID="$(docker run -d -p 127.0.0.1::8000 "$IMAGE_TAG" 2>/tmp/pre-submit-docker-run.err || true)"251  [ -n "$DOCKER_CONTAINER_ID" ] || {252    cat /tmp/pre-submit-docker-run.err253    die "docker run failed"254  }255 256  HOST_PORT="$(docker port "$DOCKER_CONTAINER_ID" 8000/tcp | tail -n 1 | awk -F: '{print $NF}')"257  [ -n "$HOST_PORT" ] || die "could not resolve mapped host port for container"258 259  HEALTH_OK=false260  for _ in $(seq 1 30); do261    health_code=$(curl -s -o /tmp/pre-submit-health.out -w "%{http_code}" \262      "http://127.0.0.1:${HOST_PORT}/health" --max-time 3 || printf "000")263    if [ "$health_code" = "200" ]; then264      HEALTH_OK=true265      break266    fi267    sleep 1268  done269  [ "$HEALTH_OK" = true ] || {270    docker logs "$DOCKER_CONTAINER_ID" | tail -n 50271    die "container did not become healthy on /health"272  }273 274  reset_code=$(curl -s -o /tmp/pre-submit-reset.out -w "%{http_code}" -X POST \275    -H "Content-Type: application/json" -d '{}' \276    "http://127.0.0.1:${HOST_PORT}/reset" --max-time 10 || printf "000")277  [ "$reset_code" = "200" ] || {278    docker logs "$DOCKER_CONTAINER_ID" | tail -n 50279    die "container /reset returned HTTP $reset_code"280  }281 282  pass "Containerized execution check passed (/health and /reset)"283 284  docker rm -f "$DOCKER_CONTAINER_ID" >/dev/null 2>&1 || true285  DOCKER_CONTAINER_ID=""286fi287 288log "Step 6/8: Environment variable checks"289if [ "$SKIP_INFERENCE" = true ]; then290  log "SKIPPED -- --skip-inference enabled"291else292  [ -n "${API_BASE_URL:-}" ] || die "API_BASE_URL is not set"293  [ -n "${MODEL_NAME:-}" ] || die "MODEL_NAME is not set"294  [ -n "${HF_TOKEN:-}" ] || die "HF_TOKEN is not set"295  pass "Required API_BASE_URL / MODEL_NAME / HF_TOKEN are set"296fi297 298log "Step 7/8: Baseline reproducibility (inference.py)"299if [ "$SKIP_INFERENCE" = true ]; then300  log "SKIPPED -- --skip-inference enabled"301else302  run_with_timeout "$INFERENCE_TIMEOUT" "$PYTHON_BIN" inference.py >"$INFERENCE_OUT_FILE" 2>&1 || {303    tail -n 80 "$INFERENCE_OUT_FILE"304    die "inference.py failed or timed out"305  }306  pass "inference.py completed within timeout"307fi308 309log "Step 8/8: Structured logs + task/grader checks"310if [ "$SKIP_INFERENCE" = true ]; then311  log "SKIPPED -- --skip-inference enabled"312else313  "$PYTHON_BIN" - "$INFERENCE_OUT_FILE" <<'PY'314import json315import sys316from pathlib import Path317 318path = Path(sys.argv[1])319text = path.read_text(encoding='utf-8', errors='replace').splitlines()320 321starts = []322ends = []323step_count = 0324 325for line in text:326    line = line.strip()327    if line.startswith('[START] '):328        payload = json.loads(line[len('[START] '):])329        starts.append(payload)330    elif line.startswith('[STEP] '):331        json.loads(line[len('[STEP] '):])332        step_count += 1333    elif line.startswith('[END] '):334        payload = json.loads(line[len('[END] '):])335        ends.append(payload)336 337if len(starts) < 3:338    raise SystemExit('Expected at least 3 [START] task logs')339 340unique_tasks = {str(s.get('task', '')) for s in starts if s.get('task')}341if len(unique_tasks) < 3:342    raise SystemExit('Expected at least 3 unique tasks in [START] logs')343 344if len(ends) != len(starts):345    raise SystemExit('Mismatch between [START] and [END] log counts')346 347if step_count == 0:348    raise SystemExit('No [STEP] logs found')349 350for i, end in enumerate(ends, start=1):351    score = float(end.get('score', -1.0))352    rewards = end.get('rewards', [])353    if not (0.0 <= score <= 1.0):354        raise SystemExit(f'END #{i} score out of range [0,1]: {score}')355    if not isinstance(rewards, list):356        raise SystemExit(f'END #{i} rewards must be a list')357    for r in rewards:358        rv = float(r)359        if not (-1.0 <= rv <= 1.0):360            raise SystemExit(f'END #{i} step reward out of sanity range [-1,1]: {rv}')361 362print('Structured logs and task/grader checks passed')363PY364  pass "Structured [START]/[STEP]/[END] logs and score-range checks passed"365fi366 367log "All checks passed. Submission is ready."368