JetBrains-Research/commit-labeling

1import json2import os3import random4import uuid5from datetime import datetime6from itertools import chain7 8import gradio as gr9 10from data_loader import load_data11 12HF_TOKEN = os.environ.get('HF_TOKEN')13HF_DATASET = os.environ.get('HF_DATASET')14N_QUESTIONS = 515 16data, models = load_data()17 18n_samples = len(data)19n_models = len(models)20 21saver = gr.HuggingFaceDatasetSaver(HF_TOKEN, HF_DATASET, private=True)22 23 24def convert_diff_to_unified(diff):25    result = "\n".join(26        [27            f'--- {modified_file["old_path"]}\n'28            f'+++ {modified_file["new_path"]}\n'29            f'{modified_file["diff"]}'30            for modified_file in diff31        ]32    )33 34    return result35 36 37def get_diff2html_view(raw_diff):38    html = f"""39    <div style='width:100%; height:1400px; overflow:auto; position: relative'>40        <div id='diff-raw' hidden>{raw_diff}</div> 41        <div class="d2h-view-wrapper">42            <div id='diff-view'></div>43        </div>44    </div>45    """46 47    return html48 49 50def get_github_link_md(repo, hash):51    return f'[See the commit on Github](https://github.com/{repo}/commit/{hash})'52 53 54def update_commit_view(sample_ind):55    if sample_ind >= n_samples:56        return None57 58    record = data[sample_ind]59 60    diff_view = get_diff2html_view(convert_diff_to_unified(json.loads(record['mods'])))61 62    repo_val = record['repo']63    hash_val = record['hash']64    github_link_md = get_github_link_md(repo_val, hash_val)65 66    diff_loaded_timestamp = datetime.now().isoformat()67 68    models_shuffled = models[:]69    random.shuffle(models_shuffled)70 71    commit_messages = tuple(record[model] for model in models_shuffled)72 73    return (74        github_link_md, diff_view, repo_val, hash_val, diff_loaded_timestamp,75        n_forms_submitted) + commit_messages + tuple(76        models_shuffled)77 78 79def reset_answers():80    return (None,) * (N_QUESTIONS * n_models)81 82 83def reset_submit_buttons():84    return tuple(gr.Button(value="Submit", interactive=True) for _ in range(n_models))85 86 87def reset_continue_button():88    return gr.Button(value=f"0/{n_models} forms submitted", interactive=False)89 90 91def next_sample(current_sample_ind, shuffled_idx):92    if current_sample_ind == n_samples:93        return None94 95    current_sample_ind += 196    updated_view = update_commit_view(shuffled_idx[current_sample_ind])97    return (current_sample_ind,) + updated_view + reset_answers() + reset_submit_buttons() + (reset_continue_button(),)98 99 100with open("head.html") as head_file:101    head_html = head_file.read()102 103force_light_theme_js_func = """104function refresh() {105    const url = new URL(window.location);106 107    if (url.searchParams.get('__theme') !== 'light') {108        url.searchParams.set('__theme', 'light');109        window.location.href = url.href;110    }111}112"""113 114with gr.Blocks(theme=gr.themes.Soft(), head=head_html, css="style_overrides.css",115               js=force_light_theme_js_func) as application:116    repo_val = gr.Textbox(interactive=False, label='repo', visible=False)117    hash_val = gr.Textbox(interactive=False, label='hash', visible=False)118    shuffled_idx_val = gr.JSON(visible=False)119 120    with gr.Row():121        with gr.Accordion("Help"):122            with open("survey_guide.md") as content_file:123                gr.Markdown(content_file.read())124 125    with gr.Row():126        current_sample_sld = gr.Slider(minimum=0, maximum=n_samples, step=1,127                                       value=0,128                                       interactive=False,129                                       label='sample_ind',130                                       info=f"Samples labeled/skipped (out of {n_samples})",131                                       show_label=False,132                                       container=False,133                                       scale=5)134        with gr.Column(scale=1):135            skip_btn = gr.Button("Skip the current sample")136    with gr.Row():137        with gr.Column(scale=2):138            github_link = gr.Markdown()139            diff_view = gr.HTML()140        with gr.Column(scale=1):141            # commit_msg = []142            # is_correct = []143            # # has_what = []144            # # has_why = []145            # is_not_verbose = []146            # # has_headline = []147            # easy_to_read = []148            # overall_rating = []149            # comments = []150            # model_name = []151 152            commit_msgs = []153            questions = []154            model_names = []155            submit_buttons = []156 157            SCALE = list(range(1, 6))158 159            for model_ind in range(n_models):160                with gr.Tab(f"Variant #{model_ind + 1}"):161                    commit_msgs.append(gr.TextArea(label="Commit message (can be scrollable)",162                                                   interactive=False,163                                                   ))164                    gr.Markdown("## Please, rate your level of agreement with each statement\n"165                                "\n"166                                "*1 - strongly disagree, 2 - disagree, 3 - not sure, 4 - agree, 5 - strongly agree*")167 168                    model_questions = []169 170                    model_questions.append(gr.Radio(171                        info='The information provided in the commit message is consistent with the code changes.',172                        label=f'is_correct',173                        show_label=False,174                        choices=SCALE,175                        interactive=True))176 177                    # model_questions.append(gr.Radio(178                    #     info='The commit message answers the question of WHAT changes have been made.',179                    #     label=f'has_what',180                    #     show_label=False,181                    #     choices=SCALE,182                    #     interactive=True))183                    #184                    # model_questions.append(gr.Radio(185                    #     info='The commit message answers the question of WHY these changes have been made.',186                    #     label=f'has_why',187                    #     show_label=False,188                    #     choices=SCALE,189                    #     interactive=True))190 191                    model_questions.append(gr.Radio(192                        info='The commit message cannot be substantially shortened without loss of important '193                             'information.',194                        label=f'is_not_verbose',195                        show_label=False,196                        choices=SCALE,197                        interactive=True))198 199                    # model_questions.append(gr.Radio(200                    #     info='The commit message includes a short headline that provides a good overview of the '201                    #          'changes.',202                    #     label=f'has_headline',203                    #     show_label=False,204                    #     choices=SCALE,205                    #     interactive=True))206 207                    model_questions.append(gr.Radio(208                        info='The commit message is easy to read and to understand.',209                        label=f'easy_to_read',210                        show_label=False,211                        choices=SCALE,212                        interactive=True))213 214                    model_questions.append(gr.Radio(215                        info='Please, describe your overall impression of the commit message (1 - very bad, 5 - very '216                             'good)',217                        label=f'overall_rating',218                        show_label=False,219                        choices=SCALE,220                        interactive=True))221 222                    model_questions.append(gr.Textbox(223                        info='Additional comments on the commit message',224                        label=f'comments',225                        show_label=False,226                        interactive=True))227 228                    assert len(model_questions) == N_QUESTIONS229 230                    questions.append(model_questions)231                    model_names.append(gr.Textbox(interactive=False, label=f'model', visible=False))232 233                    submit_buttons.append(gr.Button(value="Submit"))234 235            n_forms_submitted = gr.Number(visible=False, value=0, precision=0)236            continue_btn = reset_continue_button()237 238            session_val = gr.Textbox(info='Session', interactive=False, container=True, show_label=False,239                                     label='session')240 241            with gr.Row(visible=False):242                sample_loaded_timestamp = gr.Textbox(info="Sample loaded", label='loaded_ts', interactive=False,243                                                     container=True, show_label=False)244                sample_submitted_timestamp = gr.Textbox(info="Current time",245                                                        interactive=False, container=True, show_label=False,246                                                        value=lambda: datetime.now().isoformat(), every=1.0,247                                                        label='submitted_ts')248 249    commit_view = [250        github_link,251        diff_view,252        repo_val,253        hash_val,254        sample_loaded_timestamp,255        n_forms_submitted,256        *commit_msgs,257        *model_names,258    ]259 260    feedback_metadata = [261        session_val,262        repo_val,263        hash_val,264        sample_loaded_timestamp,265        sample_submitted_timestamp266    ]267 268    saver.setup([current_sample_sld] + feedback_metadata + questions[0] + [model_names[0], ], "feedback")269 270    questions_list = list(chain.from_iterable(questions))271 272    skip_btn.click(next_sample, inputs=[current_sample_sld, shuffled_idx_val],273                   outputs=[current_sample_sld] + commit_view + questions_list + submit_buttons + [continue_btn])274 275    continue_btn.click(next_sample, inputs=[current_sample_sld, shuffled_idx_val],276                       outputs=[current_sample_sld] + commit_view + questions_list + submit_buttons + [continue_btn])277 278 279    def submit_for_model(current_sample, n_forms_submitted_val, *args):280        saver.flag((current_sample,) + args)281        n_forms_submitted_val += 1282        all_forms_submitted = n_forms_submitted_val == n_models283        return (gr.Button(value="Submitted", interactive=False),284                n_forms_submitted_val,285                gr.Button(286                    "Next sample" if all_forms_submitted else f"{n_forms_submitted_val}/{n_models} forms submitted",287                    interactive=all_forms_submitted))288 289 290    for model_ind in range(n_models):291        submit_buttons[model_ind].click(292            submit_for_model,293            inputs=[current_sample_sld, n_forms_submitted] + feedback_metadata + questions[model_ind] + [294                model_names[model_ind], ],295            outputs=[submit_buttons[model_ind], n_forms_submitted, continue_btn]296        )297 298 299    def init_session(current_sample):300        session = str(uuid.uuid4())301        shuffled_idx = list(range(n_samples))302        random.shuffle(shuffled_idx)303        return (session, shuffled_idx) + update_commit_view(shuffled_idx[current_sample])304 305 306    application.load(init_session,307                     inputs=[current_sample_sld],308                     outputs=[session_val, shuffled_idx_val] + commit_view, )309 310application.launch()311