tokenintelligence/LiveCodeBench-SnapShot-0406

1import json2import numpy as np3import matplotlib.pyplot as plt4 5def load_graded(path):6    with open(path) as f:7        data = json.load(f)8    return [item["graded_list"] for item in data]9 10datasets = [11    ("Qwen3-4B-Base", "output/Qwen3-4B-Base/Scenario.codegeneration_16_0.6_eval_all.json"),12    ("ftajwar (MaxRL 1000)", "output/ftajwar/qwen3_4B_Base_MaxRL_Polaris_1000_steps/Scenario.codegeneration_16_0.6_eval_all.json"),13    ("ftajwar (GRPO 1000)", "output/ftajwar/qwen3_4B_Base_GRPO_Polaris_1000_steps/Scenario.codegeneration_16_0.6_eval_all.json"),14]15 16def bootstrap_pass_at_k(graded_lists, k, n_bootstrap=10000, rng=None):17    """Bootstrapping estimator for pass@k averaged over problems."""18    if rng is None:19        rng = np.random.default_rng(42)20    problem_scores = []21    for outcomes in graded_lists:22        outcomes_arr = np.array(outcomes, dtype=bool)23        n = len(outcomes_arr)24        # Sample k indices with replacement, check if any pass25        samples = rng.integers(0, n, size=(n_bootstrap, k))26        any_pass = outcomes_arr[samples].any(axis=1)27        problem_scores.append(any_pass.mean())28    return np.mean(problem_scores)29 30k_values = [1, 2, 4, 8, 16]31fig, ax = plt.subplots(figsize=(8, 5))32 33all_values = []34for label, path in datasets:35    graded = load_graded(path)36    rng = np.random.default_rng(42)37    pass_at_k = [bootstrap_pass_at_k(graded, k, rng=rng) for k in k_values]38    all_values.extend(pass_at_k)39    print(f"\n{label}:")40    for k, v in zip(k_values, pass_at_k):41        print(f"  pass@{k}: {v:.4f}")42    line, = ax.plot(k_values, pass_at_k, marker="o", linewidth=2, markersize=8, label=label)43    for k, v in zip(k_values, pass_at_k):44        ax.annotate(f"{v:.3f}", (k, v), textcoords="offset points", xytext=(5, 6),45                    fontsize=8, color=line.get_color())46 47ax.set_xscale("log", base=2)48ax.set_xticks(k_values)49ax.set_xticklabels([str(k) for k in k_values])50ax.set_xlabel("k", fontsize=12)51ax.set_ylabel("pass@k", fontsize=12)52ax.set_title("pass@k (bootstrapping estimator)", fontsize=13)53ax.set_ylim(0, max(all_values) * 1.3)54ax.legend(fontsize=10)55ax.grid(True, alpha=0.3)56plt.tight_layout()57plt.savefig("instruct.png", dpi=150)58print("\nSaved instruct.png")59