tokenintelligence/LiveCodeBench-SnapShot-0406

1import os2import json3 4from lcb_runner.runner.parser import get_args5from lcb_runner.utils.scenarios import Scenario6from lcb_runner.utils.path_utils import get_output_path7from lcb_runner.evaluation import extract_instance_results8from lcb_runner.runner.scenario_router import (9    build_prompt_benchmark,10    sort_and_extract_save_results,11    get_metrics,12)13 14 15def main():16    args = get_args()17 18    benchmark, _ = build_prompt_benchmark(args)19 20    with open(args.custom_output_file, "r") as f:21        custom_outputs = json.load(f)22        assert isinstance(custom_outputs, list)23        assert len(custom_outputs) == len(benchmark), f"{len(custom_outputs)} != {len(benchmark)}"24        if isinstance(custom_outputs[0], list):25            ## custom outputs must list[list[str]]26            ## list of extracted outputs per question27            ## sorted by the benchmark question_id, test_id, id depending on the scenario28 29            assert all(30                isinstance(custom_output, list) for custom_output in custom_outputs31            )32        elif isinstance(custom_outputs[0], dict):33            ## custom outputs must list[dict[str, Any]]34            ## list of extracted outputs per question35            ## for codegeneration and selfrepair scenario -- `code_list` and `question_id` are required36            ## for testoutputprediction -- `pred_list`, `question_id`, `test_id` are required 37            ## for codeexecution -- `pred_list`, `id` are required 38            ## code_list/pred_list is a list of extracted answers (code or assertions) for a question39 40            assert all(41                isinstance(custom_output, dict) for custom_output in custom_outputs42            )43            if args.scenario in [Scenario.codegeneration, Scenario.selfrepair]:44                custom_outputs = [45                    custom_output["code_list"]46                    for custom_output in sorted(47                        custom_outputs, key=lambda x: str(x["question_id"])48                    )49                ]50            elif args.scenario == Scenario.testoutputprediction:51                custom_outputs = [52                    custom_output['pred_list']53                    for custom_output in sorted(54                        custom_outputs, key=lambda x: (str(x["question_id"]), str(x['test_id']))55                    )56                ]57            elif args.scenario == Scenario.codeexecution:58                custom_outputs = [59                    custom_output['pred_list']60                    for custom_output in sorted(61                        custom_outputs, key=lambda x: int(x.id.split("_")[1])62                    )63                ]64 65    save_results = [66        instance.insert_output(custom_output, custom_output)67        for instance, custom_output in zip(benchmark, custom_outputs)68    ]69 70    save_results, combined_results = sort_and_extract_save_results(71        args.scenario, save_results72    )73 74    metrics = get_metrics(args.scenario, args, benchmark, combined_results)75    graded = extract_instance_results(metrics[1])76 77    if args.scenario == Scenario.codegeneration:78        metadatas = metrics[2]79        save_eval_results = [80            instance.insert_output_evaluation(81                outputs_list, extracted_list, graded_list, metadata=meta82            )83            for instance, (outputs_list, extracted_list), graded_list, meta in zip(84                benchmark, combined_results, graded, metadatas85            )86        ]87    else:88        save_eval_results = [89            instance.insert_output_evaluation(90                outputs_list, extracted_list, graded_list91            )92            for instance, (outputs_list, extracted_list), graded_list in zip(93                benchmark, combined_results, graded94            )95        ]96    97 98    if args.custom_output_save_name is None:99        output_path = args.custom_output_file[:-5] + f"_{args.scenario.value}_output.json"100    else:101        output_path = get_output_path(args.custom_output_save_name, args)102 103    with open(output_path, "w") as f:104        json.dump(save_results, f, indent=4)105 106 107    with open(output_path.replace(".json", "_eval.json"), "w") as f:108        json.dump(metrics, f, indent=4)109 110    with open(output_path.replace(".json", "_eval_all.json"), "w") as f:111        json.dump(save_eval_results, f, indent=4)112 113if __name__ == "__main__":114    main()115