open-llm-leaderboard/open_llm_leaderboard

1from dataclasses import dataclass2 3# These classes are for user facing column names, to avoid having to change them 4# all around the code when a modif is needed 5@dataclass6class ColumnContent:7    name: str8    type: str 9    displayed_by_default: bool 10    hidden: bool = False11 12def fields(raw_class):13    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]14 15@dataclass(frozen=True)16class AutoEvalColumn: # Auto evals column17    model_type_symbol = ColumnContent("T", "str", True)18    model = ColumnContent("Model", "markdown", True)19    average = ColumnContent("Average ⬆️", "number", True)20    arc = ColumnContent("ARC", "number", True)21    hellaswag = ColumnContent("HellaSwag", "number", True)22    mmlu = ColumnContent("MMLU", "number", True)23    truthfulqa = ColumnContent("TruthfulQA", "number", True)24    model_type = ColumnContent("Type", "str", False)25    precision = ColumnContent("Precision", "str", False) #, True)26    license = ColumnContent("Hub License", "str", False)27    params = ColumnContent("#Params (B)", "number", False)28    likes = ColumnContent("Hub ❤️", "number", False)29    revision = ColumnContent("Model sha", "str", False, False)30    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)31 32@dataclass(frozen=True)33class EloEvalColumn: # Elo evals column34    model = ColumnContent("Model", "markdown", True)35    gpt4 = ColumnContent("GPT-4 (all)", "number", True)36    human_all = ColumnContent("Human (all)", "number", True)37    human_instruct = ColumnContent("Human (instruct)", "number", True)38    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)39 40 41@dataclass(frozen=True)42class EvalQueueColumn: # Queue column43    model = ColumnContent("model", "markdown", True)44    revision = ColumnContent("revision", "str", True)45    private = ColumnContent("private", "bool", True)46    precision = ColumnContent("precision", "str", True)47    weight_type = ColumnContent("weight_type", "str", "Original")48    status = ColumnContent("status", "str", True)49 50LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]51 52 53KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"54VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"55OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"56DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"57MODEL_PAGE = "https://huggingface.co/models"58LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"59VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"60ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"61 62 63def model_hyperlink(link, model_name):64    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'65 66 67def make_clickable_model(model_name):68    link = f"https://huggingface.co/{model_name}"69 70    if model_name in LLAMAS:71        link = LLAMA_LINK72        model_name = model_name.split("/")[1]73    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":74        link = VICUNA_LINK75        model_name = "stable-vicuna-13b"76    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":77        link = ALPACA_LINK78        model_name = "alpaca-13b"79    if model_name == "dolly-12b":80        link = DOLLY_LINK81    elif model_name == "vicuna-13b":82        link = VICUNA_LINK83    elif model_name == "koala-13b":84        link = KOALA_LINK85    elif model_name == "oasst-12b":86        link = OASST_LINK87    #else:88    #    link = MODEL_PAGE89  90    return model_hyperlink(link, model_name)91 92def styled_error(error):93    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"94 95def styled_warning(warn):96    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"97 98def styled_message(message):99    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"