Alignment-Lab-AI/orcaleaderboard

1import os2import logging3import time4import datetime5import gradio as gr6import datasets7from huggingface_hub import snapshot_download8from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns9import plotly.graph_objects as go10 11from src.display.about import (12    CITATION_BUTTON_LABEL,13    CITATION_BUTTON_TEXT,14    FAQ_TEXT,15    INTRODUCTION_TEXT,16    LLM_BENCHMARKS_TEXT,17    TITLE,18)19from src.display.css_html_js import custom_css20from src.display.utils import (21    BENCHMARK_COLS,22    COLS,23    EVAL_COLS,24    AutoEvalColumn,25    fields,26)27from src.envs import (28    EVAL_REQUESTS_PATH,29    AGGREGATED_REPO,30    QUEUE_REPO,31    REPO_ID,32    HF_HOME,33)34from src.populate import get_evaluation_queue_df, get_leaderboard_df35from src.tools.plots import create_plot_df, create_scores_df36 37# Configure logging38logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")39 40# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.41# This controls whether a full initialization should be performed.42DO_FULL_INIT = os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"43LAST_UPDATE_LEADERBOARD = datetime.datetime.now()44 45def time_diff_wrapper(func):46    def wrapper(*args, **kwargs):47        start_time = time.time()48        result = func(*args, **kwargs)49        end_time = time.time()50        diff = end_time - start_time51        logging.info(f"Time taken for {func.__name__}: {diff} seconds")52        return result53 54    return wrapper55 56@time_diff_wrapper57def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5):58    """Download dataset with exponential backoff retries."""59    attempt = 060    while attempt < max_attempts:61        try:62            logging.info(f"Downloading {repo_id} to {local_dir}")63            snapshot_download(64                repo_id=repo_id,65                local_dir=local_dir,66                repo_type=repo_type,67                tqdm_class=None,68                etag_timeout=30,69                max_workers=8,70            )71            logging.info("Download successful")72            return73        except Exception as e:74            wait_time = backoff_factor**attempt75            logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s")76            time.sleep(wait_time)77            attempt += 178    raise Exception(f"Failed to download {repo_id} after {max_attempts} attempts")79 80def get_latest_data_leaderboard(leaderboard_initial_df = None):81    current_time = datetime.datetime.now()82    global LAST_UPDATE_LEADERBOARD83    if current_time - LAST_UPDATE_LEADERBOARD < datetime.timedelta(minutes=10) and leaderboard_initial_df is not None:84        return leaderboard_initial_df85    LAST_UPDATE_LEADERBOARD = current_time86    leaderboard_dataset = datasets.load_dataset(87        AGGREGATED_REPO, 88        "default", 89        split="train", 90        cache_dir=HF_HOME, 91        download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 92        verification_mode="no_checks"93    )94 95    leaderboard_df = get_leaderboard_df(96        leaderboard_dataset=leaderboard_dataset, 97        cols=COLS,98        benchmark_cols=BENCHMARK_COLS,99    )100 101    return leaderboard_df102 103def get_latest_data_queue():104    eval_queue_dfs = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)105    return eval_queue_dfs106 107def init_space():108    """Initializes the application space, loading only necessary data."""109    if DO_FULL_INIT:110        # These downloads only occur on full initialization111        download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)112 113    # Always redownload the leaderboard DataFrame114    leaderboard_df = get_latest_data_leaderboard()115 116    # Evaluation queue DataFrame retrieval is independent of initialization detail level117    eval_queue_dfs = get_latest_data_queue()118 119    return leaderboard_df, eval_queue_dfs120 121# Initialize the space122leaderboard_df, eval_queue_dfs = init_space()123finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = eval_queue_dfs124 125# Data processing for plots now only on demand in the respective Gradio tab126def load_and_create_plots():127    plot_df = create_plot_df(create_scores_df(leaderboard_df))128    return plot_df129 130def create_metric_plot_obj(df, metrics, title="Metrics Over Time"):131    """Create plot with Open-Orca models highlighted in purple"""132    fig = go.Figure()133    134    # Add traces for each metric135    for metric in metrics:136        # Get the model names for this metric137        model_names = df[f"{metric}_model"].tolist()138        139        # Create masks for Open-Orca and non-Open-Orca models140        is_open_orca = ["Open-Orca" in str(model) for model in model_names]141        142        # Add trace for non-Open-Orca models143        fig.add_trace(144            go.Scatter(145                x=df[df.index[~is_open_orca]],146                y=df[metric][~is_open_orca],147                name=metric,148                mode='lines+markers',149                line=dict(width=2),150                marker=dict(size=8),151                hovertemplate=(152                    "Date: %{x}<br>"153                    "Score: %{y:.2f}<br>"154                    "Model: %{text}<br>"155                ),156                text=[model_names[i] for i, flag in enumerate(is_open_orca) if not flag]157            )158        )159        160        # Add trace for Open-Orca models with purple color and larger markers161        if any(is_open_orca):162            fig.add_trace(163                go.Scatter(164                    x=df[df.index[is_open_orca]],165                    y=df[metric][is_open_orca],166                    name=f"{metric} (Open-Orca)",167                    mode='lines+markers',168                    line=dict(color='purple', width=3),169                    marker=dict(170                        color='purple',171                        size=12,172                        symbol='star'173                    ),174                    hovertemplate=(175                        "Date: %{x}<br>"176                        "Score: %{y:.2f}<br>"177                        "Model: %{text}<br>"178                    ),179                    text=[model_names[i] for i, flag in enumerate(is_open_orca) if flag]180                )181            )182    183    # Update layout184    fig.update_layout(185        title=title,186        xaxis_title="Date",187        yaxis_title="Score",188        hovermode='x unified',189        showlegend=True,190        legend=dict(191            yanchor="top",192            y=0.99,193            xanchor="left",194            x=0.01195        )196    )197    198    return fig199 200def init_leaderboard(dataframe):201    return Leaderboard(202        value = dataframe,203        datatype=[c.type for c in fields(AutoEvalColumn)],204        select_columns=SelectColumns(205            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],206            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],207            label="Select Columns to Display:",208        ),209        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.fullname.name, AutoEvalColumn.license.name],210        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],211        filter_columns=[212            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),213            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),214            ColumnFilter(215                AutoEvalColumn.params.name,216                type="slider",217                min=0.01,218                max=150,219                label="Select the number of parameters (B)",220            ),221            ColumnFilter(222                AutoEvalColumn.still_on_hub.name, type="boolean", label="Private or deleted", default=True223            ),224            ColumnFilter(225                AutoEvalColumn.merged.name, type="boolean", label="Contains a merge/moerge", default=True226            ),227            ColumnFilter(AutoEvalColumn.moe.name, type="boolean", label="MoE", default=False),228            ColumnFilter(AutoEvalColumn.not_flagged.name, type="boolean", label="Flagged", default=True),229        ],230        bool_checkboxgroup_label="Hide models",231        interactive=False,232    )233 234demo = gr.Blocks(css=custom_css)235with demo:236    gr.HTML(TITLE)237    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")238 239    with gr.Tabs(elem_classes="tab-buttons") as tabs:240        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):241            leaderboard = init_leaderboard(leaderboard_df)242 243        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):244            with gr.Row():245                with gr.Column():246                    plot_df = load_and_create_plots()247                    chart = create_metric_plot_obj(248                        plot_df,249                        [AutoEvalColumn.average.name],250                        title="Average of Top Scores and Human Baseline Over Time (from last update)",251                    )252                    gr.Plot(value=chart, min_width=500)253                with gr.Column():254                    plot_df = load_and_create_plots()255                    chart = create_metric_plot_obj(256                        plot_df,257                        BENCHMARK_COLS,258                        title="Top Scores and Human Baseline Over Time (from last update)",259                    )260                    gr.Plot(value=chart, min_width=500)261 262        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):263            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")264 265        with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=4):266            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")267 268    with gr.Row():269        with gr.Accordion("📙 Citation", open=False):270            citation_button = gr.Textbox(271                value=CITATION_BUTTON_TEXT,272                label=CITATION_BUTTON_LABEL,273                lines=20,274                elem_id="citation-button",275                show_copy_button=True,276            )277 278    demo.load(fn=get_latest_data_leaderboard, inputs=[leaderboard], outputs=[leaderboard])279 280demo.queue(default_concurrency_limit=40).launch()