tokenintelligence/LiveCodeBench-SnapShot-0406

1import json2import numpy as np3import matplotlib.pyplot as plt4 5def load_graded(path):6    with open(path) as f:7        data = json.load(f)8    return [item["graded_list"] for item in data]9 10def bootstrap_pass_at_k(graded_lists, k, n_bootstrap=10000, rng=None):11    if rng is None:12        rng = np.random.default_rng(42)13    problem_scores = []14    for outcomes in graded_lists:15        outcomes_arr = np.array(outcomes, dtype=bool)16        n = len(outcomes_arr)17        samples = rng.integers(0, n, size=(n_bootstrap, k))18        any_pass = outcomes_arr[samples].any(axis=1)19        problem_scores.append(any_pass.mean())20    return np.mean(problem_scores)21 22with open("output/base_model_evals.json") as f:23    base_evals = json.load(f)24one_shot = base_evals["results"]["generic_base_one_shot"]25 26maxrl_steps = [100, 200, 300, 400]27tailrl_steps = [100, 200, 300, 400]28k_values = [1, 2, 4, 8, 16]29left_k_values = [1, 16]30 31maxrl_data = {}32for step in maxrl_steps:33    path = f"output/maxrl_binary_{step}/Scenario.codegeneration_16_0.6_eval_all.json"34    graded = load_graded(path)35    rng = np.random.default_rng(42)36    maxrl_data[step] = {k: bootstrap_pass_at_k(graded, k, rng=rng) for k in k_values}37 38tailrl_data = {}39for step in tailrl_steps:40    path = f"output/tailrl_cont_{step}/Scenario.codegeneration_16_0.6_eval_all.json"41    graded = load_graded(path)42    rng = np.random.default_rng(42)43    tailrl_data[step] = {k: bootstrap_pass_at_k(graded, k, rng=rng) for k in k_values}44 45results_json = {46    "base_one_shot": {f"pass@{k}": one_shot[f"pass@{k}"] for k in k_values},47    "maxrl_binary": {str(step): {f"pass@{k}": float(v) for k, v in ks.items()} for step, ks in maxrl_data.items()},48    "tailrl_cont": {str(step): {f"pass@{k}": float(v) for k, v in ks.items()} for step, ks in tailrl_data.items()},49}50results_path = "output/pass_at_k_results.json"51with open(results_path, "w") as f:52    json.dump(results_json, f, indent=2)53print(f"Saved {results_path}")54 55fig, (ax_left, ax_right) = plt.subplots(1, 2, figsize=(14, 5))56 57maxrl_colors = {'p@1': 'steelblue', 'p@16': 'navy'}58tailrl_colors = {'p@1': 'darkorange', 'p@16': 'saddlebrown'}59markers = {1: 'o', 16: 's'}60 61# --- Left plot: p@1 and p@16 vs steps ---62for k in left_k_values:63    vals_maxrl = [maxrl_data[s][k] for s in maxrl_steps]64    vals_tailrl = [tailrl_data[s][k] for s in tailrl_steps]65    ax_left.plot(maxrl_steps, vals_maxrl, marker=markers[k], color=maxrl_colors[f'p@{k}'],66                 linewidth=2, markersize=7, label=f"maxrl p@{k}")67    ax_left.plot(tailrl_steps, vals_tailrl, marker=markers[k], color=tailrl_colors[f'p@{k}'],68                 linewidth=2, markersize=7, linestyle='--', label=f"tailrl p@{k}")69 70for k, ls in zip(left_k_values, ['-', '--']):71    ax_left.axhline(one_shot[f"pass@{k}"], color='gray', linewidth=1.5,72                    linestyle=ls, label=f"base one-shot p@{k}")73 74ax_left.set_xlabel("Training Steps", fontsize=12)75ax_left.set_ylabel("pass@k", fontsize=12)76ax_left.set_title("Performance vs Training Steps", fontsize=13)77ax_left.set_xticks(sorted(set(maxrl_steps + tailrl_steps)))78ax_left.legend(fontsize=10, loc="lower right")79ax_left.grid(True, alpha=0.3)80 81# --- Right plot: pass@k at last checkpoint (step 400) ---82last_maxrl = [maxrl_data[400][k] for k in k_values]83last_tailrl = [tailrl_data[400][k] for k in k_values]84 85base_one_shot_vals = [one_shot[f"pass@{k}"] for k in k_values]86ax_right.plot(k_values, base_one_shot_vals, marker='^', color='gray',87              linewidth=2, markersize=8, linestyle='--', label="base one-shot")88for k, v in zip(k_values, base_one_shot_vals):89    ax_right.annotate(f"{v:.3f}", (k, v), textcoords="offset points", xytext=(5, 6),90                      fontsize=8, color='gray')91 92ax_right.plot(k_values, last_maxrl, marker='o', color='steelblue',93              linewidth=2, markersize=8, label="maxrl_binary_400")94ax_right.plot(k_values, last_tailrl, marker='s', color='darkorange',95              linewidth=2, markersize=8, label="tailrl_cont_400")96for k, v in zip(k_values, last_maxrl):97    ax_right.annotate(f"{v:.3f}", (k, v), textcoords="offset points", xytext=(5, 6),98                      fontsize=8, color='steelblue')99for k, v in zip(k_values, last_tailrl):100    ax_right.annotate(f"{v:.3f}", (k, v), textcoords="offset points", xytext=(5, -14),101                      fontsize=8, color='darkorange')102 103ax_right.set_xscale("log", base=2)104ax_right.set_xticks(k_values)105ax_right.set_xticklabels([str(k) for k in k_values])106ax_right.set_xlabel("k", fontsize=12)107ax_right.set_ylabel("pass@k", fontsize=12)108ax_right.set_title("pass@k at Last Checkpoint", fontsize=13)109all_right = last_maxrl + last_tailrl + base_one_shot_vals110ax_right.set_ylim(min(all_right) * 0.9, max(all_right) * 1.15)111ax_right.legend(fontsize=10)112ax_right.grid(True, alpha=0.3)113 114plt.tight_layout()115plt.savefig("output/pass_at_k_comparison.png", dpi=150)116print("Saved output/pass_at_k_comparison.png")117