JetBrains-Research/commit-message-editing-visualization
0
1import gradio as gr2import numpy as np3import pandas as pd4import plotly.graph_objects as go5from datasets import load_dataset6from evaluate.utils import parse_readme7from scipy.stats import gaussian_kde, spearmanr8 9import generate_annotated_diffs10from api_wrappers import hf_data_loader11from generation_steps.metrics_analysis import AGGR_METRICS, edit_distance_fn12 13colors = {14 "Expert-labeled": "#C19C0B",15 "Synthetic Backward": "#913632",16 "Synthetic Forward": "#58136a",17 "Full": "#000000",18}19 20METRICS = {21 "Edit Distance": "editdist",22 "Edit Similarity": "editsim",23 "BLEU": "bleu",24 "METEOR": "meteor",25 "ROUGE-1": "rouge1",26 "ROUGE-2": "rouge2",27 "ROUGE-L": "rougeL",28 "BERTScore": "bertscore",29 "ChrF": "chrF",30}31 32 33df_related = generate_annotated_diffs.data_with_annotated_diffs()34 35 36def golden():37 return df_related.loc[(df_related["G_type"] == "initial") & (df_related["E_type"] == "expert_labeled")].reset_index(38 drop=True39 )40 41 42def backward():43 return df_related.loc[44 (df_related["G_type"] == "synthetic_backward") & (df_related["E_type"] == "expert_labeled")45 ].reset_index(drop=True)46 47 48def forward():49 return df_related.loc[50 (df_related["G_type"] == "initial") & (df_related["E_type"] == "synthetic_forward")51 ].reset_index(drop=True)52 53 54def forward_from_backward():55 return df_related.loc[56 (df_related.G_type == "synthetic_backward")57 & (df_related.E_type.isin(["synthetic_forward", "synthetic_forward_from_backward"]))58 ].reset_index(drop=True)59 60 61n_diffs_manual = len(golden())62n_diffs_synthetic_backward = len(backward())63n_diffs_synthetic_forward = len(forward())64n_diffs_synthetic_forward_backward = len(forward_from_backward())65 66 67def update_dataset_view(diff_idx, df):68 diff_idx -= 169 return (70 df.iloc[diff_idx]["annotated_diff"],71 df.iloc[diff_idx]["commit_msg_start"] if "commit_msg_start" in df.columns else df.iloc[diff_idx]["G_text"],72 df.iloc[diff_idx]["commit_msg_end"] if "commit_msg_end" in df.columns else df.iloc[diff_idx]["E_text"],73 f"https://github.com/{df.iloc[diff_idx]['repo']}/commit/{df.iloc[diff_idx]['hash']}",74 )75 76 77def update_dataset_view_manual(diff_idx):78 return update_dataset_view(diff_idx, golden())79 80 81def update_dataset_view_synthetic_backward(diff_idx):82 return update_dataset_view(diff_idx, backward())83 84 85def update_dataset_view_synthetic_forward(diff_idx):86 return update_dataset_view(diff_idx, forward())87 88 89def update_dataset_view_synthetic_forward_backward(diff_idx):90 return update_dataset_view(diff_idx, forward_from_backward())91 92 93def number_of_pairs_plot():94 related_plot_dict = {95 "Full": df_related,96 "Synthetic Backward": backward(),97 "Synthetic Forward": pd.concat([forward(), forward_from_backward()], axis=0, ignore_index=True),98 "Expert-labeled": golden(),99 }100 101 df_unrelated = hf_data_loader.load_synthetic_as_pandas()102 df_unrelated = df_unrelated.loc[~df_unrelated.is_related].copy()103 unrelated_plot_dict = {104 "Full": df_unrelated,105 "Synthetic Backward": df_unrelated.loc[106 (df_unrelated["G_type"] == "synthetic_backward")107 & (~df_unrelated.E_type.isin(["synthetic_forward", "synthetic_forward_from_backward"]))108 ],109 "Synthetic Forward": df_unrelated.loc[110 ((df_unrelated["G_type"] == "initial") & (df_unrelated["E_type"] == "synthetic_forward"))111 | (112 (df_unrelated["G_type"] == "synthetic_backward")113 & (df_unrelated["E_type"].isin(["synthetic_forward", "synthetic_forward_from_backward"]))114 )115 ],116 "Expert-labeled": df_unrelated.loc[117 (df_unrelated.G_type == "initial") & (df_unrelated.E_type == "expert_labeled")118 ],119 }120 121 traces = []122 123 for split in related_plot_dict.keys():124 related_count = len(related_plot_dict[split])125 unrelated_count = len(unrelated_plot_dict[split])126 127 traces.append(128 go.Bar(129 name=f"{split} - Related pairs",130 x=[split],131 y=[related_count],132 marker=dict(133 color=colors[split],134 ),135 )136 )137 138 traces.append(139 go.Bar(140 name=f"{split} - Conditionally independent pairs",141 x=[split],142 y=[unrelated_count],143 marker=dict(144 color=colors[split],145 pattern=dict(146 shape="/", # Crosses147 fillmode="overlay",148 solidity=0.5,149 ),150 ),151 )152 )153 154 fig = go.Figure(data=traces)155 156 fig.update_layout(157 barmode="stack",158 bargap=0.2,159 xaxis=dict(title="Split", showgrid=True, gridcolor="lightgrey"),160 yaxis=dict(title="Number of Examples", showgrid=True, gridcolor="lightgrey"),161 legend=dict(title="Pair Type", orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),162 plot_bgcolor="rgba(0,0,0,0)",163 paper_bgcolor="rgba(0,0,0,0)",164 width=1100,165 )166 return fig167 168 169def edit_distance_plot():170 df_edit_distance = {171 "Full": [edit_distance_fn(pred=row["G_text"], ref=row["E_text"]) for _, row in df_related.iterrows()],172 "Synthetic Backward": [173 edit_distance_fn(pred=row["G_text"], ref=row["E_text"]) for _, row in backward().iterrows()174 ],175 "Synthetic Forward": [176 edit_distance_fn(pred=row["G_text"], ref=row["E_text"])177 for _, row in pd.concat([forward(), forward_from_backward()], axis=0, ignore_index=True).iterrows()178 ],179 "Expert-labeled": [edit_distance_fn(pred=row["G_text"], ref=row["E_text"]) for _, row in golden().iterrows()],180 }181 traces = []182 183 for key in df_edit_distance:184 kde_x = np.linspace(0, 1200, 1000)185 kde = gaussian_kde(df_edit_distance[key])186 kde_line = go.Scatter(x=kde_x, y=kde(kde_x), mode="lines", name=key, line=dict(color=colors[key], width=5))187 traces.append(kde_line)188 189 fig = go.Figure(data=traces)190 191 fig.update_layout(192 bargap=0.1,193 xaxis=dict(title=dict(text="Edit Distance"), range=[0, 1200], showgrid=True, gridcolor="lightgrey"),194 yaxis=dict(195 title=dict(text="Probability Density"),196 range=[0, 0.004],197 showgrid=True,198 gridcolor="lightgrey",199 tickvals=[0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.0035, 0.004],200 tickformat=".4f",201 ),202 plot_bgcolor="rgba(0,0,0,0)",203 paper_bgcolor="rgba(0,0,0,0)",204 width=1100,205 )206 return fig207 208 209def get_correlations_table(online_metric_name: str) -> pd.DataFrame:210 df = load_dataset(211 "JetBrains-Research/synthetic-commit-msg-edits", "all_pairs_with_metrics_other_online_metrics", split="train"212 ).to_pandas()213 corr_df = (214 df.loc[~df.is_related]215 .groupby(["G_text", "G_type", "hash", "repo"] + [f"online_{online_metric_name}"])216 .apply(lambda g: g.to_dict(orient="records"), include_groups=False)217 .reset_index(name="unrelated_pairs")218 .copy()219 )220 _ = corr_df.copy()221 for metric in AGGR_METRICS:222 if metric in ["editdist"]:223 _[metric] = _.unrelated_pairs.apply(lambda pairs: min(pair[metric] for pair in pairs))224 else:225 _[metric] = _.unrelated_pairs.apply(lambda pairs: max(pair[metric] for pair in pairs))226 227 results = []228 229 for metric in AGGR_METRICS:230 x = _[metric].to_numpy()231 y = _[f"online_{online_metric_name}"].to_numpy()232 corr, p_value = spearmanr(x, y)233 results.append({"metric": metric, "corr": corr, "p_value": p_value})234 235 __ = pd.DataFrame(results)236 __["p_value"] = ["< 0.05" if p < 0.05 else p for p in __.p_value]237 __["corr_abs"] = abs(__["corr"])238 __["corr"] = __["corr"].round(2)239 __["metric"] = __["metric"].map({v: k for k, v in METRICS.items()})240 return (241 __.sort_values(by=["corr_abs"], ascending=False)242 .drop(columns=["corr_abs"])243 .rename(columns={"metric": "Metric m", "corr": "Correlation Q(m, m*)", "p_value": "p-value"})244 )245 246 247force_light_theme_js_func = """248function refresh() {249 const url = new URL(window.location);250 251 if (url.searchParams.get('__theme') !== 'light') {252 url.searchParams.set('__theme', 'light');253 window.location.href = url.href;254 }255}256"""257 258if __name__ == "__main__":259 with gr.Blocks(theme=gr.themes.Soft(), js=force_light_theme_js_func) as application:260 gr.Markdown(parse_readme("README.md"))261 262 def dataset_view_tab(n_items):263 slider = gr.Slider(minimum=1, maximum=n_items, step=1, value=1, label=f"Sample number (total: {n_items})")264 265 diff_view = gr.Highlightedtext(combine_adjacent=True, color_map={"+": "green", "-": "red"})266 start_view = gr.Textbox(interactive=False, label="Initial message G", container=True)267 end_view = gr.Textbox(interactive=False, label="Edited message E", container=True)268 link_view = gr.Markdown()269 270 view = [diff_view, start_view, end_view, link_view]271 272 return slider, view273 274 with gr.Tab("Examples Exploration"):275 with gr.Tab("Manual"):276 slider_manual, view_manual = dataset_view_tab(n_diffs_manual)277 278 slider_manual.change(update_dataset_view_manual, inputs=slider_manual, outputs=view_manual)279 280 with gr.Tab("Synthetic Backward"):281 slider_synthetic_backward, view_synthetic_backward = dataset_view_tab(n_diffs_synthetic_backward)282 283 slider_synthetic_backward.change(284 update_dataset_view_synthetic_backward,285 inputs=slider_synthetic_backward,286 outputs=view_synthetic_backward,287 )288 289 with gr.Tab("Synthetic Forward (from initial)"):290 slider_synthetic_forward, view_synthetic_forward = dataset_view_tab(n_diffs_synthetic_forward)291 292 slider_synthetic_forward.change(293 update_dataset_view_synthetic_forward,294 inputs=slider_synthetic_forward,295 outputs=view_synthetic_forward,296 )297 298 with gr.Tab("Synthetic Forward (from backward)"):299 slider_synthetic_forward_backward, view_synthetic_forward_backward = dataset_view_tab(300 n_diffs_synthetic_forward_backward301 )302 303 slider_synthetic_forward_backward.change(304 update_dataset_view_synthetic_forward_backward,305 inputs=slider_synthetic_forward_backward,306 outputs=view_synthetic_forward_backward,307 )308 309 with gr.Tab("Dataset Statistics"):310 gr.Markdown("## Number of examples per split")311 312 number_of_pairs_gr_plot = gr.Plot(number_of_pairs_plot, label=None)313 314 gr.Markdown("## Edit Distance Distribution (w/o PyCharm Logs)")315 316 edit_distance_gr_plot = gr.Plot(edit_distance_plot(), label=None)317 318 with gr.Tab("Experimental Results"):319 gr.Markdown(320 "Here, we provide the additional experimental results with different text similarity metrics used as the target online metric, "321 "in addition to edit distance between generated messages G and their edited counterparts E."322 )323 324 gr.Markdown(325 "Please, select one of the available metrics **m*** below to see the correlations **Q(m, m\*)** of offline text similarity metrics with **m*** as an online metric."326 )327 328 for metric in METRICS:329 with gr.Tab(metric):330 gr.Markdown(331 f"The table below presents the correlation coefficients **Q(m, m\*)** where {metric} is used as an online metric **m***."332 )333 334 result_df = get_correlations_table(METRICS[metric])335 gr.DataFrame(result_df)336 337 application.load(update_dataset_view_manual, inputs=slider_manual, outputs=view_manual)338 339 application.load(340 update_dataset_view_synthetic_backward, inputs=slider_synthetic_backward, outputs=view_synthetic_backward341 )342 343 application.load(344 update_dataset_view_synthetic_forward, inputs=slider_synthetic_forward, outputs=view_synthetic_forward345 )346 347 application.load(348 update_dataset_view_synthetic_forward_backward,349 inputs=slider_synthetic_forward_backward,350 outputs=view_synthetic_forward_backward,351 )352 353 application.launch()354