codekingpro/portable-devtools

1import copy2from collections import defaultdict3from dataclasses import dataclass4from typing import Any5 6from huggingface_hub.utils import logging, yaml_dump7 8 9logger = logging.get_logger(__name__)10 11 12@dataclass13class EvalResult:14    """15    Flattened representation of individual evaluation results found in model-index of Model Cards.16 17    For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1.18 19    Args:20        task_type (`str`):21            The task identifier. Example: "image-classification".22        dataset_type (`str`):23            The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets.24        dataset_name (`str`):25            A pretty name for the dataset. Example: "Common Voice (French)".26        metric_type (`str`):27            The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics.28        metric_value (`Any`):29            The metric value. Example: 0.9 or "20.0 ± 1.2".30        task_name (`str`, *optional*):31            A pretty name for the task. Example: "Speech Recognition".32        dataset_config (`str`, *optional*):33            The name of the dataset configuration used in `load_dataset()`.34            Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info:35            https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name36        dataset_split (`str`, *optional*):37            The split used in `load_dataset()`. Example: "test".38        dataset_revision (`str`, *optional*):39            The revision (AKA Git Sha) of the dataset used in `load_dataset()`.40            Example: 5503434ddd753f426f4b38109466949a1217c2bb41        dataset_args (`dict[str, Any]`, *optional*):42            The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}`43        metric_name (`str`, *optional*):44            A pretty name for the metric. Example: "Test WER".45        metric_config (`str`, *optional*):46            The name of the metric configuration used in `load_metric()`.47            Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.48            See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations49        metric_args (`dict[str, Any]`, *optional*):50            The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 451        verified (`bool`, *optional*):52            Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.53        verify_token (`str`, *optional*):54            A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.55        source_name (`str`, *optional*):56            The name of the source of the evaluation result. Example: "Open LLM Leaderboard".57        source_url (`str`, *optional*):58            The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard".59    """60 61    # Required62 63    # The task identifier64    # Example: automatic-speech-recognition65    task_type: str66 67    # The dataset identifier68    # Example: common_voice. Use dataset id from https://hf.co/datasets69    dataset_type: str70 71    # A pretty name for the dataset.72    # Example: Common Voice (French)73    dataset_name: str74 75    # The metric identifier76    # Example: wer. Use metric id from https://hf.co/metrics77    metric_type: str78 79    # Value of the metric.80    # Example: 20.0 or "20.0 ± 1.2"81    metric_value: Any82 83    # Optional84 85    # A pretty name for the task.86    # Example: Speech Recognition87    task_name: str | None = None88 89    # The name of the dataset configuration used in `load_dataset()`.90    # Example: fr in `load_dataset("common_voice", "fr")`.91    # See the `datasets` docs for more info:92    # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name93    dataset_config: str | None = None94 95    # The split used in `load_dataset()`.96    # Example: test97    dataset_split: str | None = None98 99    # The revision (AKA Git Sha) of the dataset used in `load_dataset()`.100    # Example: 5503434ddd753f426f4b38109466949a1217c2bb101    dataset_revision: str | None = None102 103    # The arguments passed during `Metric.compute()`.104    # Example for `bleu`: max_order: 4105    dataset_args: dict[str, Any] | None = None106 107    # A pretty name for the metric.108    # Example: Test WER109    metric_name: str | None = None110 111    # The name of the metric configuration used in `load_metric()`.112    # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.113    # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations114    metric_config: str | None = None115 116    # The arguments passed during `Metric.compute()`.117    # Example for `bleu`: max_order: 4118    metric_args: dict[str, Any] | None = None119 120    # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.121    verified: bool | None = None122 123    # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.124    verify_token: str | None = None125 126    # The name of the source of the evaluation result.127    # Example: Open LLM Leaderboard128    source_name: str | None = None129 130    # The URL of the source of the evaluation result.131    # Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard132    source_url: str | None = None133 134    @property135    def unique_identifier(self) -> tuple:136        """Returns a tuple that uniquely identifies this evaluation."""137        return (138            self.task_type,139            self.dataset_type,140            self.dataset_config,141            self.dataset_split,142            self.dataset_revision,143        )144 145    def is_equal_except_value(self, other: "EvalResult") -> bool:146        """147        Return True if `self` and `other` describe exactly the same metric but with a148        different value.149        """150        for key, _ in self.__dict__.items():151            if key == "metric_value":152                continue153            # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`,154            # so we exclude it here in the comparison.155            if key != "verify_token" and getattr(self, key) != getattr(other, key):156                return False157        return True158 159    def __post_init__(self) -> None:160        if self.source_name is not None and self.source_url is None:161            raise ValueError("If `source_name` is provided, `source_url` must also be provided.")162 163 164@dataclass165class CardData:166    """Structure containing metadata from a RepoCard.167 168    [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`].169 170    Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data171    (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not172    inherit from `dict` to allow this export step.173    """174 175    def __init__(self, ignore_metadata_errors: bool = False, **kwargs):176        self.__dict__.update(kwargs)177 178    def to_dict(self):179        """Converts CardData to a dict.180 181        Returns:182            `dict`: CardData represented as a dictionary ready to be dumped to a YAML183            block for inclusion in a README.md file.184        """185 186        data_dict = copy.deepcopy(self.__dict__)187        self._to_dict(data_dict)188        return {key: value for key, value in data_dict.items() if value is not None}189 190    def _to_dict(self, data_dict):191        """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place.192 193        Args:194            data_dict (`dict`): The raw dict representation of the card data.195        """196        pass197 198    def to_yaml(self, line_break=None, original_order: list[str] | None = None) -> str:199        """Dumps CardData to a YAML block for inclusion in a README.md file.200 201        Args:202            line_break (str, *optional*):203                The line break to use when dumping to yaml.204 205        Returns:206            `str`: CardData represented as a YAML block.207        """208        if original_order:209            self.__dict__ = {210                k: self.__dict__[k]211                for k in original_order + list(set(self.__dict__.keys()) - set(original_order))212                if k in self.__dict__213            }214        return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip()215 216    def __repr__(self):217        return repr(self.__dict__)218 219    def __str__(self):220        return self.to_yaml()221 222    def get(self, key: str, default: Any = None) -> Any:223        """Get value for a given metadata key."""224        value = self.__dict__.get(key)225        return default if value is None else value226 227    def pop(self, key: str, default: Any = None) -> Any:228        """Pop value for a given metadata key."""229        return self.__dict__.pop(key, default)230 231    def __getitem__(self, key: str) -> Any:232        """Get value for a given metadata key."""233        return self.__dict__[key]234 235    def __setitem__(self, key: str, value: Any) -> None:236        """Set value for a given metadata key."""237        self.__dict__[key] = value238 239    def __contains__(self, key: str) -> bool:240        """Check if a given metadata key is set."""241        return key in self.__dict__242 243    def __len__(self) -> int:244        """Return the number of metadata keys set."""245        return len(self.__dict__)246 247 248def _validate_eval_results(249    eval_results: EvalResult | list[EvalResult] | None,250    model_name: str | None,251) -> list[EvalResult]:252    if eval_results is None:253        return []254    if isinstance(eval_results, EvalResult):255        eval_results = [eval_results]256    if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results):257        raise ValueError(258            f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}."259        )260    if model_name is None:261        raise ValueError("Passing `eval_results` requires `model_name` to be set.")262    return eval_results263 264 265class ModelCardData(CardData):266    """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md267 268    Args:269        base_model (`str` or `list[str]`, *optional*):270            The identifier of the base model from which the model derives. This is applicable for example if your model is a271            fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs272            if your model derives from multiple models). Defaults to None.273        datasets (`Union[str, list[str]]`, *optional*):274            Dataset or list of datasets that were used to train this model. Should be a dataset ID275            found on https://hf.co/datasets. Defaults to None.276        eval_results (`Union[list[EvalResult], EvalResult]`, *optional*):277            List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided,278            `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`.279        language (`Union[str, list[str]]`, *optional*):280            Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or281            639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`.282        library_name (`str`, *optional*):283            Name of library used by this model. Example: keras or any library from284            https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts.285            Defaults to None.286        license (`str`, *optional*):287            License of this model. Example: apache-2.0 or any license from288            https://huggingface.co/docs/hub/repositories-licenses. Defaults to None.289        license_name (`str`, *optional*):290            Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`.291            Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead.292        license_link (`str`, *optional*):293            Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`.294            Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead.295        metrics (`list[str]`, *optional*):296            List of metrics used to evaluate this model. Should be a metric name that can be found297            at https://hf.co/metrics. Example: 'accuracy'. Defaults to None.298        model_name (`str`, *optional*):299            A name for this model. It is used along with300            `eval_results` to construct the `model-index` within the card's metadata. The name301            you supply here is what will be used on PapersWithCode's leaderboards. If None is provided302            then the repo name is used as a default. Defaults to None.303        pipeline_tag (`str`, *optional*):304            The pipeline tag associated with the model. Example: "text-classification".305        tags (`list[str]`, *optional*):306            List of tags to add to your model that can be used when filtering on the Hugging307            Face Hub. Defaults to None.308        ignore_metadata_errors (`str`):309            If True, errors while parsing the metadata section will be ignored. Some information might be lost during310            the process. Use it at your own risk.311        kwargs (`dict`, *optional*):312            Additional metadata that will be added to the model card. Defaults to None.313 314    Example:315        ```python316        >>> from huggingface_hub import ModelCardData317        >>> card_data = ModelCardData(318        ...     language="en",319        ...     license="mit",320        ...     library_name="timm",321        ...     tags=['image-classification', 'resnet'],322        ... )323        >>> card_data.to_dict()324        {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']}325 326        ```327    """328 329    def __init__(330        self,331        *,332        base_model: str | list[str] | None = None,333        datasets: str | list[str] | None = None,334        eval_results: list[EvalResult] | None = None,335        language: str | list[str] | None = None,336        library_name: str | None = None,337        license: str | None = None,338        license_name: str | None = None,339        license_link: str | None = None,340        metrics: list[str] | None = None,341        model_name: str | None = None,342        pipeline_tag: str | None = None,343        tags: list[str] | None = None,344        ignore_metadata_errors: bool = False,345        **kwargs,346    ):347        self.base_model = base_model348        self.datasets = datasets349        self.eval_results = eval_results350        self.language = language351        self.library_name = library_name352        self.license = license353        self.license_name = license_name354        self.license_link = license_link355        self.metrics = metrics356        self.model_name = model_name357        self.pipeline_tag = pipeline_tag358        self.tags = _to_unique_list(tags)359 360        model_index = kwargs.pop("model-index", None)361        if model_index:362            try:363                model_name, eval_results = model_index_to_eval_results(model_index)364                self.model_name = model_name365                self.eval_results = eval_results366            except (KeyError, TypeError) as error:367                if ignore_metadata_errors:368                    logger.warning("Invalid model-index. Not loading eval results into CardData.")369                else:370                    raise ValueError(371                        f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass"372                        " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:"373                        " some information will be lost. Use it at your own risk."374                    )375 376        super().__init__(**kwargs)377 378        if self.eval_results:379            try:380                self.eval_results = _validate_eval_results(self.eval_results, self.model_name)381            except Exception as e:382                if ignore_metadata_errors:383                    logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.")384                else:385                    raise ValueError(f"Failed to validate eval_results: {e}") from e386 387    def _to_dict(self, data_dict):388        """Format the internal data dict. In this case, we convert eval results to a valid model index"""389        if self.eval_results is not None:390            data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results)  # type: ignore391            del data_dict["eval_results"], data_dict["model_name"]392 393 394class DatasetCardData(CardData):395    """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md396 397    Args:398        language (`list[str]`, *optional*):399            Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or400            639-3 code (two/three letters), or a special value like "code", "multilingual".401        license (`Union[str, list[str]]`, *optional*):402            License(s) of this dataset. Example: apache-2.0 or any license from403            https://huggingface.co/docs/hub/repositories-licenses.404        annotations_creators (`Union[str, list[str]]`, *optional*):405            How the annotations for the dataset were created.406            Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'.407        language_creators (`Union[str, list[str]]`, *optional*):408            How the text-based data in the dataset was created.409            Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other'410        multilinguality (`Union[str, list[str]]`, *optional*):411            Whether the dataset is multilingual.412            Options are: 'monolingual', 'multilingual', 'translation', 'other'.413        size_categories (`Union[str, list[str]]`, *optional*):414            The number of examples in the dataset. Options are: 'n<1K', '1K<n<10K', '10K<n<100K',415            '100K<n<1M', '1M<n<10M', '10M<n<100M', '100M<n<1B', '1B<n<10B', '10B<n<100B', '100B<n<1T', 'n>1T', and 'other'.416        source_datasets (`list[str]]`, *optional*):417            Indicates whether the dataset is an original dataset or extended from another existing dataset.418            Options are: 'original' and 'extended'.419        task_categories (`Union[str, list[str]]`, *optional*):420            What categories of task does the dataset support?421        task_ids (`Union[str, list[str]]`, *optional*):422            What specific tasks does the dataset support?423        paperswithcode_id (`str`, *optional*):424            ID of the dataset on PapersWithCode.425        pretty_name (`str`, *optional*):426            A more human-readable name for the dataset. (ex. "Cats vs. Dogs")427        train_eval_index (`dict`, *optional*):428            A dictionary that describes the necessary spec for doing evaluation on the Hub.429            If not provided, it will be gathered from the 'train-eval-index' key of the kwargs.430        config_names (`Union[str, list[str]]`, *optional*):431            A list of the available dataset configs for the dataset.432    """433 434    def __init__(435        self,436        *,437        language: str | list[str] | None = None,438        license: str | list[str] | None = None,439        annotations_creators: str | list[str] | None = None,440        language_creators: str | list[str] | None = None,441        multilinguality: str | list[str] | None = None,442        size_categories: str | list[str] | None = None,443        source_datasets: list[str] | None = None,444        task_categories: str | list[str] | None = None,445        task_ids: str | list[str] | None = None,446        paperswithcode_id: str | None = None,447        pretty_name: str | None = None,448        train_eval_index: dict | None = None,449        config_names: str | list[str] | None = None,450        ignore_metadata_errors: bool = False,451        **kwargs,452    ):453        self.annotations_creators = annotations_creators454        self.language_creators = language_creators455        self.language = language456        self.license = license457        self.multilinguality = multilinguality458        self.size_categories = size_categories459        self.source_datasets = source_datasets460        self.task_categories = task_categories461        self.task_ids = task_ids462        self.paperswithcode_id = paperswithcode_id463        self.pretty_name = pretty_name464        self.config_names = config_names465 466        # TODO - maybe handle this similarly to EvalResult?467        self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None)468        super().__init__(**kwargs)469 470    def _to_dict(self, data_dict):471        data_dict["train-eval-index"] = data_dict.pop("train_eval_index")472 473 474class SpaceCardData(CardData):475    """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md476 477    To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference.478 479    Args:480        title (`str`, *optional*)481            Title of the Space.482        sdk (`str`, *optional*)483            SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`).484        sdk_version (`str`, *optional*)485            Version of the used SDK (if Gradio/Streamlit sdk).486        python_version (`str`, *optional*)487            Python version used in the Space (if Gradio/Streamlit sdk).488        app_file (`str`, *optional*)489            Path to your main application file (which contains either gradio or streamlit Python code, or static html code).490            Path is relative to the root of the repository.491        app_port (`str`, *optional*)492            Port on which your application is running. Used only if sdk is `docker`.493        license (`str`, *optional*)494            License of this model. Example: apache-2.0 or any license from495            https://huggingface.co/docs/hub/repositories-licenses.496        duplicated_from (`str`, *optional*)497            ID of the original Space if this is a duplicated Space.498        models (list[`str`], *optional*)499            List of models related to this Space. Should be a dataset ID found on https://hf.co/models.500        datasets (`list[str]`, *optional*)501            List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets.502        tags (`list[str]`, *optional*)503            List of tags to add to your Space that can be used when filtering on the Hub.504        ignore_metadata_errors (`str`):505            If True, errors while parsing the metadata section will be ignored. Some information might be lost during506            the process. Use it at your own risk.507        kwargs (`dict`, *optional*):508            Additional metadata that will be added to the space card.509 510    Example:511        ```python512        >>> from huggingface_hub import SpaceCardData513        >>> card_data = SpaceCardData(514        ...     title="Dreambooth Training",515        ...     license="mit",516        ...     sdk="gradio",517        ...     duplicated_from="multimodalart/dreambooth-training"518        ... )519        >>> card_data.to_dict()520        {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'}521        ```522    """523 524    def __init__(525        self,526        *,527        title: str | None = None,528        sdk: str | None = None,529        sdk_version: str | None = None,530        python_version: str | None = None,531        app_file: str | None = None,532        app_port: int | None = None,533        license: str | None = None,534        duplicated_from: str | None = None,535        models: list[str] | None = None,536        datasets: list[str] | None = None,537        tags: list[str] | None = None,538        ignore_metadata_errors: bool = False,539        **kwargs,540    ):541        self.title = title542        self.sdk = sdk543        self.sdk_version = sdk_version544        self.python_version = python_version545        self.app_file = app_file546        self.app_port = app_port547        self.license = license548        self.duplicated_from = duplicated_from549        self.models = models550        self.datasets = datasets551        self.tags = _to_unique_list(tags)552        super().__init__(**kwargs)553 554 555def model_index_to_eval_results(model_index: list[dict[str, Any]]) -> tuple[str, list[EvalResult]]:556    """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects.557 558    A detailed spec of the model index can be found here:559    https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1560 561    Args:562        model_index (`list[dict[str, Any]]`):563            A model index data structure, likely coming from a README.md file on the564            Hugging Face Hub.565 566    Returns:567        model_name (`str`):568            The name of the model as found in the model index. This is used as the569            identifier for the model on leaderboards like PapersWithCode.570        eval_results (`list[EvalResult]`):571            A list of `huggingface_hub.EvalResult` objects containing the metrics572            reported in the provided model_index.573 574    Example:575        ```python576        >>> from huggingface_hub.repocard_data import model_index_to_eval_results577        >>> # Define a minimal model index578        >>> model_index = [579        ...     {580        ...         "name": "my-cool-model",581        ...         "results": [582        ...             {583        ...                 "task": {584        ...                     "type": "image-classification"585        ...                 },586        ...                 "dataset": {587        ...                     "type": "beans",588        ...                     "name": "Beans"589        ...                 },590        ...                 "metrics": [591        ...                     {592        ...                         "type": "accuracy",593        ...                         "value": 0.9594        ...                     }595        ...                 ]596        ...             }597        ...         ]598        ...     }599        ... ]600        >>> model_name, eval_results = model_index_to_eval_results(model_index)601        >>> model_name602        'my-cool-model'603        >>> eval_results[0].task_type604        'image-classification'605        >>> eval_results[0].metric_type606        'accuracy'607 608        ```609    """610 611    eval_results = []612    for elem in model_index:613        name = elem["name"]614        results = elem["results"]615        for result in results:616            task_type = result["task"]["type"]617            task_name = result["task"].get("name")618            dataset_type = result["dataset"]["type"]619            dataset_name = result["dataset"]["name"]620            dataset_config = result["dataset"].get("config")621            dataset_split = result["dataset"].get("split")622            dataset_revision = result["dataset"].get("revision")623            dataset_args = result["dataset"].get("args")624            source_name = result.get("source", {}).get("name")625            source_url = result.get("source", {}).get("url")626 627            for metric in result["metrics"]:628                metric_type = metric["type"]629                metric_value = metric["value"]630                metric_name = metric.get("name")631                metric_args = metric.get("args")632                metric_config = metric.get("config")633                verified = metric.get("verified")634                verify_token = metric.get("verifyToken")635 636                eval_result = EvalResult(637                    task_type=task_type,  # Required638                    dataset_type=dataset_type,  # Required639                    dataset_name=dataset_name,  # Required640                    metric_type=metric_type,  # Required641                    metric_value=metric_value,  # Required642                    task_name=task_name,643                    dataset_config=dataset_config,644                    dataset_split=dataset_split,645                    dataset_revision=dataset_revision,646                    dataset_args=dataset_args,647                    metric_name=metric_name,648                    metric_args=metric_args,649                    metric_config=metric_config,650                    verified=verified,651                    verify_token=verify_token,652                    source_name=source_name,653                    source_url=source_url,654                )655                eval_results.append(eval_result)656    return name, eval_results657 658 659def _remove_none(obj):660    """661    Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778662    """663    if isinstance(obj, (list, tuple, set)):664        return type(obj)(_remove_none(x) for x in obj if x is not None)665    elif isinstance(obj, dict):666        return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None)667    else:668        return obj669 670 671def eval_results_to_model_index(model_name: str, eval_results: list[EvalResult]) -> list[dict[str, Any]]:672    """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a673    valid model-index that will be compatible with the format expected by the674    Hugging Face Hub.675 676    Args:677        model_name (`str`):678            Name of the model (ex. "my-cool-model"). This is used as the identifier679            for the model on leaderboards like PapersWithCode.680        eval_results (`list[EvalResult]`):681            List of `huggingface_hub.EvalResult` objects containing the metrics to be682            reported in the model-index.683 684    Returns:685        model_index (`list[dict[str, Any]]`): The eval_results converted to a model-index.686 687    Example:688        ```python689        >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult690        >>> # Define minimal eval_results691        >>> eval_results = [692        ...     EvalResult(693        ...         task_type="image-classification",  # Required694        ...         dataset_type="beans",  # Required695        ...         dataset_name="Beans",  # Required696        ...         metric_type="accuracy",  # Required697        ...         metric_value=0.9,  # Required698        ...     )699        ... ]700        >>> eval_results_to_model_index("my-cool-model", eval_results)701        [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}]702 703        ```704    """705 706    # Metrics are reported on a unique task-and-dataset basis.707    # Here, we make a map of those pairs and the associated EvalResults.708    task_and_ds_types_map: dict[Any, list[EvalResult]] = defaultdict(list)709    for eval_result in eval_results:710        task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)711 712    # Use the map from above to generate the model index data.713    model_index_data: list[dict[str, Any]] = []714    for results in task_and_ds_types_map.values():715        # All items from `results` share same metadata716        sample_result = results[0]717        data: dict[str, Any] = {718            "task": {719                "type": sample_result.task_type,720                "name": sample_result.task_name,721            },722            "dataset": {723                "name": sample_result.dataset_name,724                "type": sample_result.dataset_type,725                "config": sample_result.dataset_config,726                "split": sample_result.dataset_split,727                "revision": sample_result.dataset_revision,728                "args": sample_result.dataset_args,729            },730            "metrics": [731                {732                    "type": result.metric_type,733                    "value": result.metric_value,734                    "name": result.metric_name,735                    "config": result.metric_config,736                    "args": result.metric_args,737                    "verified": result.verified,738                    "verifyToken": result.verify_token,739                }740                for result in results741            ],742        }743        if sample_result.source_url is not None:744            source: dict[str, str] = {745                "url": sample_result.source_url,746            }747            if sample_result.source_name is not None:748                source["name"] = sample_result.source_name749            data["source"] = source750        model_index_data.append(data)751 752    # TODO - Check if there cases where this list is longer than one?753    # Finally, the model index itself is list of dicts.754    model_index = [755        {756            "name": model_name,757            "results": model_index_data,758        }759    ]760    return _remove_none(model_index)761 762 763def _to_unique_list(tags: list[str] | None) -> list[str] | None:764    if tags is None:765        return tags766    unique_tags = []  # make tags unique + keep order explicitly767    for tag in tags:768        if tag not in unique_tags:769            unique_tags.append(tag)770    return unique_tags771