tokenintelligence/LiveCodeBench-SnapShot-0406

1import json2import numpy as np3import matplotlib.pyplot as plt4 5with open("output/maxrl_binary_400/Scenario.codegeneration_256_0.6_eval_all.json") as f:6    data = json.load(f)7 8graded_lists = [item["graded_list"] for item in data]9 10def bootstrap_pass_at_k(graded_lists, k, n_bootstrap=10000, rng=None):11    if rng is None:12        rng = np.random.default_rng(42)13    problem_scores = []14    for outcomes in graded_lists:15        outcomes_arr = np.array(outcomes, dtype=bool)16        n = len(outcomes_arr)17        samples = rng.integers(0, n, size=(n_bootstrap, k))18        any_pass = outcomes_arr[samples].any(axis=1)19        problem_scores.append(any_pass.mean())20    return np.mean(problem_scores)21 22k_values = [1, 2, 4, 8, 16, 32, 64, 128, 256]23rng = np.random.default_rng(42)24pass_at_k = [bootstrap_pass_at_k(graded_lists, k, rng=rng) for k in k_values]25 26print("maxrl_binary_400 pass@k:")27for k, v in zip(k_values, pass_at_k):28    print(f"  pass@{k}: {v:.4f}")29 30fig, ax = plt.subplots(figsize=(8, 5))31line, = ax.plot(k_values, pass_at_k, marker="o", linewidth=2, markersize=8,32                color="steelblue", label="maxrl_binary_400")33for k, v in zip(k_values, pass_at_k):34    ax.annotate(f"{v:.3f}", (k, v), textcoords="offset points", xytext=(5, 6),35                fontsize=8, color=line.get_color())36 37ax.set_xscale("log", base=2)38ax.set_xticks(k_values)39ax.set_xticklabels([str(k) for k in k_values])40ax.set_xlabel("k", fontsize=12)41ax.set_ylabel("pass@k", fontsize=12)42ax.set_title("pass@k — maxrl_binary_400 (n=256 samples)", fontsize=13)43ax.set_ylim(0, max(pass_at_k) * 1.3)44ax.legend(fontsize=10)45ax.grid(True, alpha=0.3)46plt.tight_layout()47plt.savefig("maxrl_400_pass_at_k.png", dpi=150)48print("Saved maxrl_400_pass_at_k.png")49