zkcpku/CodeAgentBench
4
1"""2Usage:3python3 qa_browser.py --share4"""5 6# import argparse7from collections import defaultdict8import re9 10import gradio as gr11import json12 13MAX_py_file_paths_depth = 314 15 16 17 18py_file_paths = {}19 20comment = {"title2comment": {}, "path2comment": {}}21model_judgments_normal = {}22model_judgments_math = {}23 24def load_py_file_paths(data_path):25 global py_file_paths26 py_file_paths = []27 with open(data_path, "r") as f:28 py_file_paths = f.readlines()29 py_file_paths = [json.loads(line) for line in py_file_paths]30 py_file_paths = {line["path"]: line['content'] for line in py_file_paths}31 return py_file_paths32 33def load_comment(data_path):34 global comment35 comment = {"title2comment": {}, "path2comment": {}}36 title2comment = {}37 path2comment = {}38 with open(data_path, "r") as f:39 data_f = f.readlines()40 data_f = [json.loads(line) for line in data_f]41 for line in data_f:42 title2comment[line["title"]] = line43 path2comment[line["class_link"]] = line44 comment["title2comment"] = title2comment45 comment["path2comment"] = path2comment46 return comment47 48def display_question(question_selector,file_path_selector, request: gr.Request):49 if question_selector == None:50 return [gr.Dropdown.update(51 value=file_path_selector52 )] + [""] * 453 54 choice = comment['title2comment'][question_selector]['class_link'].split("#")[0]55 lines = comment['title2comment'][question_selector]['class_link'].split("#")[1].replace("L","")56 57 question = comment['title2comment'][question_selector]['comment']58 code_path = choice59 test_path = comment['title2comment'][question_selector]['test_file_path']60 61 highlight_lines = [int(line) for line in lines.split("-")]62 class_link = comment['title2comment'][question_selector]['class_link']63 question_mds,code_mds = to_gradio_chat_mds(question, code_path, highlight_lines, class_link, test_path)64 return [gr.Dropdown.update(65 value=choice66 )] + question_mds + code_mds67 68def display_answer(question_selector,file_path_selector, request: gr.Request):69 choice = comment['title2comment'][question_selector]['class_link'].split("#")[0]70 lines = comment['title2comment'][question_selector]['class_link'].split("#")[1].replace("L","")71 72 question = comment['title2comment'][question_selector]['comment']73 code_path = choice74 test_path = comment['title2comment'][question_selector]['test_file_path']75 76 highlight_lines = [int(line) for line in lines.split("-")]77 class_link = comment['title2comment'][question_selector]['class_link']78 if comment['title2comment'][question_selector]['class_link'].split("#")[0] != file_path_selector:79 highlight_lines = None80 class_link = None81 test_path = None82 code_path = file_path_selector83 question_mds,code_mds = to_gradio_chat_mds(question, code_path, highlight_lines, class_link, test_path)84 return code_mds85 86# def display_answer(87# file_path_selector, request: gr.Request88# ):89# # if comment['title2comment'][question_selector]['class_link'].split("#")[0] == file_path_selector:90# # choice = question_selector91# # else:92# # choice = None93 94# code_path = file_path_selector95# question = ""96# highlight_lines = None97 98# question_mds,code_mds = to_gradio_chat_mds(question, code_path, highlight_lines)99 100# return code_mds101 102 103newline_pattern1 = re.compile("\n\n(\d+\. )")104newline_pattern2 = re.compile("\n\n(- )")105 106 107def to_gradio_chat_mds(question, code_path, highlight_lines = None, class_link = None, test_path = None):108 mds = [""] * 1 + [""] * 3109 mds[0] = question110 source_code = py_file_paths[code_path]111 if test_path:112 test_code = py_file_paths[test_path]113 else:114 test_code = [""]115 # no highlight, highlight, no highlight116 if not highlight_lines:117 # mds[1] = "".join(source_code)118 mds[1] = "#################################################################\n" + "# Please Check the `Full Code in the File` column\n" + "#################################################################\n"119 # mds[3] = ""120 else:121 highlight_lines[0] -= 1122 mds[1] = "#################################################################\n" + "# From " + class_link + "\n" + f"# From Line {highlight_lines[0]+1} to Line {highlight_lines[1]}\n" + "#################################################################\n"123 mds[1] += "".join(source_code[highlight_lines[0]:highlight_lines[1]])124 mds[2] = "".join(source_code)125 mds[3] = "".join(test_code)126 127 return mds[:1], mds[1:]128 129 130def build_pairwise_browser_tab():131 file_path_list = list(py_file_paths.keys())132 question_list = list(comment['title2comment'].keys())133 num_py_file_paths = MAX_py_file_paths_depth134 135 # Build question selector map136 with gr.Row():137 with gr.Column():138 question_selector = gr.Dropdown(139 choices=question_list,140 label="Question",141 container=False142 )143 # locate_button = gr.Button(144 # text="Locate",145 # label="Locate",146 # type="default"147 # )148 with gr.Column():149 file_path_selector = gr.Dropdown(150 choices=file_path_list,151 label="File Path",152 container=False153 )154 155 question_mds = []156 code_mds = []157 with gr.Row():158 with gr.Column():159 question_mds.append(gr.Code(language="markdown"))160 with gr.Column():161 with gr.Tab("Infilling Code"):162 # add code highlight163 code_mds.append(gr.Code(language="python",elem_id="locate_code"))164 with gr.Tab("Full Code in the File"):165 code_mds.append(gr.Code(language="python"))166 with gr.Tab("Test Code"):167 code_mds.append(gr.Code(language="python"))168 169 # Callbacks170 question_selector.change(display_question, [question_selector, file_path_selector], [file_path_selector] + question_mds + code_mds)171 file_path_selector.change(display_answer, [question_selector,file_path_selector], code_mds)172 173 return (question_selector,)174 175 176 177 178code_highlight_css = """179#chatbot .hll { background-color: #ffffcc }180#chatbot .c { color: #408080; font-style: italic }181#chatbot .err { border: 1px solid #FF0000 }182#chatbot .k { color: #008000; font-weight: bold }183#chatbot .o { color: #666666 }184#chatbot .ch { color: #408080; font-style: italic }185#chatbot .cm { color: #408080; font-style: italic }186#chatbot .cp { color: #BC7A00 }187#chatbot .cpf { color: #408080; font-style: italic }188#chatbot .c1 { color: #408080; font-style: italic }189#chatbot .cs { color: #408080; font-style: italic }190#chatbot .gd { color: #A00000 }191#chatbot .ge { font-style: italic }192#chatbot .gr { color: #FF0000 }193#chatbot .gh { color: #000080; font-weight: bold }194#chatbot .gi { color: #00A000 }195#chatbot .go { color: #888888 }196#chatbot .gp { color: #000080; font-weight: bold }197#chatbot .gs { font-weight: bold }198#chatbot .gu { color: #800080; font-weight: bold }199#chatbot .gt { color: #0044DD }200#chatbot .kc { color: #008000; font-weight: bold }201#chatbot .kd { color: #008000; font-weight: bold }202#chatbot .kn { color: #008000; font-weight: bold }203#chatbot .kp { color: #008000 }204#chatbot .kr { color: #008000; font-weight: bold }205#chatbot .kt { color: #B00040 }206#chatbot .m { color: #666666 }207#chatbot .s { color: #BA2121 }208#chatbot .na { color: #7D9029 }209#chatbot .nb { color: #008000 }210#chatbot .nc { color: #0000FF; font-weight: bold }211#chatbot .no { color: #880000 }212#chatbot .nd { color: #AA22FF }213#chatbot .ni { color: #999999; font-weight: bold }214#chatbot .ne { color: #D2413A; font-weight: bold }215#chatbot .nf { color: #0000FF }216#chatbot .nl { color: #A0A000 }217#chatbot .nn { color: #0000FF; font-weight: bold }218#chatbot .nt { color: #008000; font-weight: bold }219#chatbot .nv { color: #19177C }220#chatbot .ow { color: #AA22FF; font-weight: bold }221#chatbot .w { color: #bbbbbb }222#chatbot .mb { color: #666666 }223#chatbot .mf { color: #666666 }224#chatbot .mh { color: #666666 }225#chatbot .mi { color: #666666 }226#chatbot .mo { color: #666666 }227#chatbot .sa { color: #BA2121 }228#chatbot .sb { color: #BA2121 }229#chatbot .sc { color: #BA2121 }230#chatbot .dl { color: #BA2121 }231#chatbot .sd { color: #BA2121; font-style: italic }232#chatbot .s2 { color: #BA2121 }233#chatbot .se { color: #BB6622; font-weight: bold }234#chatbot .sh { color: #BA2121 }235#chatbot .si { color: #BB6688; font-weight: bold }236#chatbot .sx { color: #008000 }237#chatbot .sr { color: #BB6688 }238#chatbot .s1 { color: #BA2121 }239#chatbot .ss { color: #19177C }240#chatbot .bp { color: #008000 }241#chatbot .fm { color: #0000FF }242#chatbot .vc { color: #19177C }243#chatbot .vg { color: #19177C }244#chatbot .vi { color: #19177C }245#chatbot .vm { color: #19177C }246#chatbot .il { color: #666666 }247"""248# .highlight { background: #f8f8f8; }249 250table_css = """251table {252 line-height: 0em253}254"""255old_block_css = (256 code_highlight_css257 + """258pre {259 white-space: pre-wrap; /* Since CSS 2.1 */260 white-space: -moz-pre-wrap; /* Mozilla, since 1999 */261 white-space: -pre-wrap; /* Opera 4-6 */262 white-space: -o-pre-wrap; /* Opera 7 */263 word-wrap: break-word; /* Internet Explorer 5.5+ */264}265#notice_markdown th {266 display: none;267}268#notice_markdown td {269 padding-top: 8px;270 padding-bottom: 8px;271}272#leaderboard_markdown td {273 padding-top: 8px;274 padding-bottom: 8px;275}276"""277)278block_css = old_block_css + (279 """280#locate_code {281 background-color: #DEEBF7;282}283#user_question {284 background-color: #E2F0D9;285}286#reference {287 background-color: #FFF2CC;288}289#model_explanation {290 background-color: #FBE5D6;291}292"""293)294 295 296def load_demo():297 dropdown_update = gr.Dropdown.update(value=list(comment['title2comment'].keys())[0])298 return dropdown_update299 300 301def build_demo():302 with gr.Blocks(303 title="Class-level Repo Code Generation Benchmark (numpy-ml)",304 theme=gr.themes.Base(text_size=gr.themes.sizes.text_lg),305 css=block_css,306 ) as demo:307 308 (question_selector,) = build_pairwise_browser_tab()309 310 demo.load(load_demo, [], [question_selector])311 312 thanks_txt = gr.Markdown(313 """314 # Acknowledgement315 This benchmark is based on the [numpy-ml](https://github.com/ddbourgin/numpy-ml) project.316 """317 )318 319 return demo320 321 322# if __name__ == "__main__":323# parser = argparse.ArgumentParser()324# parser.add_argument("--host", type=str, default="0.0.0.0")325# parser.add_argument("--port", type=int)326# parser.add_argument("--share", action="store_true")327# parser.add_argument("--bench-name", type=str, default="mt_bench")328# args = parser.parse_args()329# print(args)330 331py_file_paths_path = "numpyml-display/all_py_content.jsonl"332comment_path = "numpyml-display/final_dataset.jsonl"333 334# Load py_file_paths335py_file_paths = load_py_file_paths(py_file_paths_path)336 337# Load answers338comment = load_comment(comment_path)339 340demo = build_demo()341# demo.queue(concurrency_count=10, status_update_rate=10, api_open=False).launch()342demo.launch()343