Alignment-Lab-AI/orcaleaderboard

1from dataclasses import dataclass, make_dataclass2from enum import Enum3import json4import logging5from datetime import datetime6import pandas as pd7 8 9# Configure logging10logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")11 12 13def parse_datetime(datetime_str):14    formats = [15        "%Y-%m-%dT%H-%M-%S.%f",  # Format with dashes16        "%Y-%m-%dT%H:%M:%S.%f",  # Standard format with colons17        "%Y-%m-%dT%H %M %S.%f",  # Spaces as separator18    ]19 20    for fmt in formats:21        try:22            return datetime.strptime(datetime_str, fmt)23        except ValueError:24            continue25    # in rare cases set unix start time for files with incorrect time (legacy files)26    logging.error(f"No valid date format found for: {datetime_str}")27    return datetime(1970, 1, 1)28 29 30def load_json_data(file_path):31    """Safely load JSON data from a file."""32    try:33        with open(file_path, "r") as file:34            return json.load(file)35    except json.JSONDecodeError:36        print(f"Error reading JSON from {file_path}")37        return None  # Or raise an exception38 39 40def fields(raw_class):41    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]42 43 44@dataclass45class Task:46    benchmark: str47    metric: str48    col_name: str49 50 51class Tasks(Enum):52    arc = Task("arc:challenge", "acc_norm", "ARC")53    hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")54    mmlu = Task("hendrycksTest", "acc", "MMLU")55    truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")56    winogrande = Task("winogrande", "acc", "Winogrande")57    gsm8k = Task("gsm8k", "acc", "GSM8K")58 59 60# These classes are for user facing column names,61# to avoid having to change them all around the code62# when a modif is needed63@dataclass(frozen=True)64class ColumnContent:65    name: str66    type: str67    displayed_by_default: bool68    hidden: bool = False69    never_hidden: bool = False70    dummy: bool = False71 72 73auto_eval_column_dict = []74# Init75auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])76auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])77# Scores78auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])79for task in Tasks:80    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])81# Model information82auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])83auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])84auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])85auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])86auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False)])87auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])88auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])89auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])90auto_eval_column_dict.append(91    ["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)]92)93auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])94auto_eval_column_dict.append(["not_flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])95auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])96auto_eval_column_dict.append(["date", ColumnContent, ColumnContent("date", "bool", False, hidden=True)])97# Dummy column for the search bar (hidden by the custom CSS)98auto_eval_column_dict.append(["fullname", ColumnContent, ColumnContent("fullname", "str", False, dummy=True)])99 100# We use make dataclass to dynamically fill the scores from Tasks101AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)102 103 104@dataclass(frozen=True)105class EvalQueueColumn:  # Queue column106    model = ColumnContent("model", "markdown", True)107    revision = ColumnContent("revision", "str", True)108    private = ColumnContent("private", "bool", True)109    precision = ColumnContent("precision", "str", True)110    weight_type = ColumnContent("weight_type", "str", "Original")111    status = ColumnContent("status", "str", True)112 113 114baseline_row = {115    AutoEvalColumn.model.name: "<p>Baseline</p>",116    AutoEvalColumn.revision.name: "N/A",117    AutoEvalColumn.precision.name: None,118    AutoEvalColumn.merged.name: False,119    AutoEvalColumn.average.name: 31.0,120    AutoEvalColumn.arc.name: 25.0,121    AutoEvalColumn.hellaswag.name: 25.0,122    AutoEvalColumn.mmlu.name: 25.0,123    AutoEvalColumn.truthfulqa.name: 25.0,124    AutoEvalColumn.winogrande.name: 50.0,125    AutoEvalColumn.gsm8k.name: 0.21,126    AutoEvalColumn.fullname.name: "baseline",127    AutoEvalColumn.model_type.name: "",128    AutoEvalColumn.not_flagged.name: False,129}130 131# Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)132# ARC human baseline is 0.80 (source: https://lab42.global/arc/)133# HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide)134# MMLU human baseline is 0.898 (source: https://openreview.net/forum?id=d7KBjmI3GmQ)135# TruthfulQA human baseline is 0.94(source: https://arxiv.org/pdf/2109.07958.pdf)136# Winogrande: https://leaderboard.allenai.org/winogrande/submissions/public137# GSM8K: paper138# Define the human baselines139human_baseline_row = {140    AutoEvalColumn.model.name: "<p>Human performance</p>",141    AutoEvalColumn.revision.name: "N/A",142    AutoEvalColumn.precision.name: None,143    AutoEvalColumn.average.name: 92.75,144    AutoEvalColumn.merged.name: False,145    AutoEvalColumn.arc.name: 80.0,146    AutoEvalColumn.hellaswag.name: 95.0,147    AutoEvalColumn.mmlu.name: 89.8,148    AutoEvalColumn.truthfulqa.name: 94.0,149    AutoEvalColumn.winogrande.name: 94.0,150    AutoEvalColumn.gsm8k.name: 100,151    AutoEvalColumn.fullname.name: "human_baseline",152    AutoEvalColumn.model_type.name: "",153    AutoEvalColumn.not_flagged.name: False,154}155 156 157@dataclass158class ModelDetails:159    name: str160    symbol: str = ""  # emoji, only for the model type161 162 163class ModelType(Enum):164    PT = ModelDetails(name="🟢 pretrained", symbol="🟢")165    CPT = ModelDetails(name="🟩 continuously pretrained", symbol="🟩")166    FT = ModelDetails(name="🔶 fine-tuned on domain-specific datasets", symbol="🔶")167    chat = ModelDetails(name="💬 chat models (RLHF, DPO, IFT, ...)", symbol="💬")168    merges = ModelDetails(name="🤝 base merges and moerges", symbol="🤝")169    Unknown = ModelDetails(name="", symbol="?")170 171    def to_str(self, separator=" "):172        return f"{self.value.symbol}{separator}{self.value.name}"173 174    @staticmethod175    def from_str(type):176        if "fine-tuned" in type or "🔶" in type:177            return ModelType.FT178        if "continously pretrained" in type or "🟩" in type:179            return ModelType.CPT180        if "pretrained" in type or "🟢" in type:181            return ModelType.PT182        if any([k in type for k in ["instruction-tuned", "RL-tuned", "chat", "🟦", "⭕", "💬"]]):183            return ModelType.chat184        if "merge" in type or "🤝" in type:185            return ModelType.merges186        return ModelType.Unknown187 188 189class WeightType(Enum):190    Adapter = ModelDetails("Adapter")191    Original = ModelDetails("Original")192    Delta = ModelDetails("Delta")193 194 195class Precision(Enum):196    float16 = ModelDetails("float16")197    bfloat16 = ModelDetails("bfloat16")198    qt_8bit = ModelDetails("8bit")199    qt_4bit = ModelDetails("4bit")200    qt_GPTQ = ModelDetails("GPTQ")201    Unknown = ModelDetails("?")202 203    def from_str(precision):204        if precision in ["torch.float16", "float16"]:205            return Precision.float16206        if precision in ["torch.bfloat16", "bfloat16"]:207            return Precision.bfloat16208        if precision in ["8bit"]:209            return Precision.qt_8bit210        if precision in ["4bit"]:211            return Precision.qt_4bit212        if precision in ["GPTQ", "None"]:213            return Precision.qt_GPTQ214        return Precision.Unknown215 216 217# Column selection218COLS = [c.name for c in fields(AutoEvalColumn)]219TYPES = [c.type for c in fields(AutoEvalColumn)]220 221EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]222EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]223 224BENCHMARK_COLS = [t.value.col_name for t in Tasks]225 226NUMERIC_INTERVALS = {227    "?": pd.Interval(-1, 0, closed="right"),228    "~1.5": pd.Interval(0, 2, closed="right"),229    "~3": pd.Interval(2, 4, closed="right"),230    "~7": pd.Interval(4, 9, closed="right"),231    "~13": pd.Interval(9, 20, closed="right"),232    "~35": pd.Interval(20, 45, closed="right"),233    "~60": pd.Interval(45, 70, closed="right"),234    "70+": pd.Interval(70, 10000, closed="right"),235}236