codekingpro/portable-devtools

1import json2import os3from pathlib import Path4from pickle import DEFAULT_PROTOCOL, PicklingError5from typing import Any6 7from packaging import version8 9from huggingface_hub import constants, snapshot_download10from huggingface_hub.hf_api import HfApi11from huggingface_hub.utils import (12    SoftTemporaryDirectory,13    get_fastai_version,14    get_fastcore_version,15    get_python_version,16)17 18from .utils import logging, validate_hf_hub_args19 20 21logger = logging.get_logger(__name__)22 23 24def _check_fastai_fastcore_versions(25    fastai_min_version: str = "2.4",26    fastcore_min_version: str = "1.3.27",27):28    """29    Checks that the installed fastai and fastcore versions are compatible for pickle serialization.30 31    Args:32        fastai_min_version (`str`, *optional*):33            The minimum fastai version supported.34        fastcore_min_version (`str`, *optional*):35            The minimum fastcore version supported.36 37    > [!TIP]38    > Raises the following error:39    >40    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)41    >       if the fastai or fastcore libraries are not available or are of an invalid version.42    """43 44    if (get_fastcore_version() or get_fastai_version()) == "N/A":45        raise ImportError(46            f"fastai>={fastai_min_version} and fastcore>={fastcore_min_version} are"47            f" required. Currently using fastai=={get_fastai_version()} and"48            f" fastcore=={get_fastcore_version()}."49        )50 51    current_fastai_version = version.Version(get_fastai_version())52    current_fastcore_version = version.Version(get_fastcore_version())53 54    if current_fastai_version < version.Version(fastai_min_version):55        raise ImportError(56            "`push_to_hub_fastai` and `from_pretrained_fastai` require a"57            f" fastai>={fastai_min_version} version, but you are using fastai version"58            f" {get_fastai_version()} which is incompatible. Upgrade with `pip install"59            " fastai==2.5.6`."60        )61 62    if current_fastcore_version < version.Version(fastcore_min_version):63        raise ImportError(64            "`push_to_hub_fastai` and `from_pretrained_fastai` require a"65            f" fastcore>={fastcore_min_version} version, but you are using fastcore"66            f" version {get_fastcore_version()} which is incompatible. Upgrade with"67            " `pip install fastcore==1.3.27`."68        )69 70 71def _check_fastai_fastcore_pyproject_versions(72    storage_folder: str,73    fastai_min_version: str = "2.4",74    fastcore_min_version: str = "1.3.27",75):76    """77    Checks that the `pyproject.toml` file in the directory `storage_folder` has fastai and fastcore versions78    that are compatible with `from_pretrained_fastai` and `push_to_hub_fastai`. If `pyproject.toml` does not exist79    or does not contain versions for fastai and fastcore, then it logs a warning.80 81    Args:82        storage_folder (`str`):83            Folder to look for the `pyproject.toml` file.84        fastai_min_version (`str`, *optional*):85            The minimum fastai version supported.86        fastcore_min_version (`str`, *optional*):87            The minimum fastcore version supported.88 89    > [!TIP]90    > Raises the following errors:91    >92    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)93    >       if the `toml` module is not installed.94    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)95    >       if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.96    """97 98    try:99        import toml100    except ModuleNotFoundError:101        raise ImportError(102            "`push_to_hub_fastai` and `from_pretrained_fastai` require the toml module."103            " Install it with `pip install toml`."104        )105 106    # Checks that a `pyproject.toml`, with `build-system` and `requires` sections, exists in the repository. If so, get a list of required packages.107    if not os.path.isfile(f"{storage_folder}/pyproject.toml"):108        logger.warning(109            "There is no `pyproject.toml` in the repository that contains the fastai"110            " `Learner`. The `pyproject.toml` would allow us to verify that your fastai"111            " and fastcore versions are compatible with those of the model you want to"112            " load."113        )114        return115    pyproject_toml = toml.load(f"{storage_folder}/pyproject.toml")116 117    if "build-system" not in pyproject_toml.keys():118        logger.warning(119            "There is no `build-system` section in the pyproject.toml of the repository"120            " that contains the fastai `Learner`. The `build-system` would allow us to"121            " verify that your fastai and fastcore versions are compatible with those"122            " of the model you want to load."123        )124        return125    build_system_toml = pyproject_toml["build-system"]126 127    if "requires" not in build_system_toml.keys():128        logger.warning(129            "There is no `requires` section in the pyproject.toml of the repository"130            " that contains the fastai `Learner`. The `requires` would allow us to"131            " verify that your fastai and fastcore versions are compatible with those"132            " of the model you want to load."133        )134        return135    package_versions = build_system_toml["requires"]136 137    # Extracts contains fastai and fastcore versions from `pyproject.toml` if available.138    # If the package is specified but not the version (e.g. "fastai" instead of "fastai=2.4"), the default versions are the highest.139    fastai_packages = [pck for pck in package_versions if pck.startswith("fastai")]140    if len(fastai_packages) == 0:141        logger.warning("The repository does not have a fastai version specified in the `pyproject.toml`.")142    # fastai_version is an empty string if not specified143    else:144        fastai_version = str(fastai_packages[0]).partition("=")[2]145        if fastai_version != "" and version.Version(fastai_version) < version.Version(fastai_min_version):146            raise ImportError(147                "`from_pretrained_fastai` requires"148                f" fastai>={fastai_min_version} version but the model to load uses"149                f" {fastai_version} which is incompatible."150            )151 152    fastcore_packages = [pck for pck in package_versions if pck.startswith("fastcore")]153    if len(fastcore_packages) == 0:154        logger.warning("The repository does not have a fastcore version specified in the `pyproject.toml`.")155    # fastcore_version is an empty string if not specified156    else:157        fastcore_version = str(fastcore_packages[0]).partition("=")[2]158        if fastcore_version != "" and version.Version(fastcore_version) < version.Version(fastcore_min_version):159            raise ImportError(160                "`from_pretrained_fastai` requires"161                f" fastcore>={fastcore_min_version} version, but you are using fastcore"162                f" version {fastcore_version} which is incompatible."163            )164 165 166README_TEMPLATE = """---167tags:168- fastai169---170 171# Amazing!172 173🥳 Congratulations on hosting your fastai model on the Hugging Face Hub!174 175# Some next steps1761. Fill out this model card with more information (see the template below and the [documentation here](https://huggingface.co/docs/hub/model-repos))!177 1782. Create a demo in Gradio or Streamlit using 🤗 Spaces ([documentation here](https://huggingface.co/docs/hub/spaces)).179 1803. Join the fastai community on the [Fastai Discord](https://discord.com/invite/YKrxeNn)!181 182Greetings fellow fastlearner 🤝! Don't forget to delete this content from your model card.183 184 185---186 187 188# Model card189 190## Model description191More information needed192 193## Intended uses & limitations194More information needed195 196## Training and evaluation data197More information needed198"""199 200PYPROJECT_TEMPLATE = f"""[build-system]201requires = ["setuptools>=40.8.0", "wheel", "python={get_python_version()}", "fastai={get_fastai_version()}", "fastcore={get_fastcore_version()}"]202build-backend = "setuptools.build_meta:__legacy__"203"""204 205 206def _create_model_card(repo_dir: Path):207    """208    Creates a model card for the repository.209 210    Args:211        repo_dir (`Path`):212            Directory where model card is created.213    """214    readme_path = repo_dir / "README.md"215 216    if not readme_path.exists():217        with readme_path.open("w", encoding="utf-8") as f:218            f.write(README_TEMPLATE)219 220 221def _create_model_pyproject(repo_dir: Path):222    """223    Creates a `pyproject.toml` for the repository.224 225    Args:226        repo_dir (`Path`):227            Directory where `pyproject.toml` is created.228    """229    pyproject_path = repo_dir / "pyproject.toml"230 231    if not pyproject_path.exists():232        with pyproject_path.open("w", encoding="utf-8") as f:233            f.write(PYPROJECT_TEMPLATE)234 235 236def _save_pretrained_fastai(237    learner,238    save_directory: str | Path,239    config: dict[str, Any] | None = None,240):241    """242    Saves a fastai learner to `save_directory` in pickle format using the default pickle protocol for the version of python used.243 244    Args:245        learner (`Learner`):246            The `fastai.Learner` you'd like to save.247        save_directory (`str` or `Path`):248            Specific directory in which you want to save the fastai learner.249        config (`dict`, *optional*):250            Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'.251 252    > [!TIP]253    > Raises the following error:254    >255    >     - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)256    >       if the config file provided is not a dictionary.257    """258    _check_fastai_fastcore_versions()259 260    os.makedirs(save_directory, exist_ok=True)261 262    # if the user provides config then we update it with the fastai and fastcore versions in CONFIG_TEMPLATE.263    if config is not None:264        if not isinstance(config, dict):265            raise RuntimeError(f"Provided config should be a dict. Got: '{type(config)}'")266        path = os.path.join(save_directory, constants.CONFIG_NAME)267        with open(path, "w") as f:268            json.dump(config, f)269 270    _create_model_card(Path(save_directory))271    _create_model_pyproject(Path(save_directory))272 273    # learner.export saves the model in `self.path`.274    learner.path = Path(save_directory)275    os.makedirs(save_directory, exist_ok=True)276    try:277        learner.export(278            fname="model.pkl",279            pickle_protocol=DEFAULT_PROTOCOL,280        )281    except PicklingError:282        raise PicklingError(283            "You are using a lambda function, i.e., an anonymous function. `pickle`"284            " cannot pickle function objects and requires that all functions have"285            " names. One possible solution is to name the function."286        )287 288 289@validate_hf_hub_args290def from_pretrained_fastai(291    repo_id: str,292    revision: str | None = None,293):294    """295    Load pretrained fastai model from the Hub or from a local directory.296 297    Args:298        repo_id (`str`):299            The location where the pickled fastai.Learner is. It can be either of the two:300                - Hosted on the Hugging Face Hub. E.g.: 'espejelomar/fatai-pet-breeds-classification' or 'distilgpt2'.301                  You can add a `revision` by appending `@` at the end of `repo_id`. E.g.: `dbmdz/bert-base-german-cased@main`.302                  Revision is the specific model version to use. Since we use a git-based system for storing models and other303                  artifacts on the Hugging Face Hub, it can be a branch name, a tag name, or a commit id.304                - Hosted locally. `repo_id` would be a directory containing the pickle and a pyproject.toml305                  indicating the fastai and fastcore versions used to build the `fastai.Learner`. E.g.: `./my_model_directory/`.306        revision (`str`, *optional*):307            Revision at which the repo's files are downloaded. See documentation of `snapshot_download`.308 309    Returns:310        The `fastai.Learner` model in the `repo_id` repo.311    """312    _check_fastai_fastcore_versions()313 314    # Load the `repo_id` repo.315    # `snapshot_download` returns the folder where the model was stored.316    # `cache_dir` will be the default '/root/.cache/huggingface/hub'317    if not os.path.isdir(repo_id):318        storage_folder = snapshot_download(319            repo_id=repo_id,320            revision=revision,321            library_name="fastai",322            library_version=get_fastai_version(),323        )324    else:325        storage_folder = repo_id326 327    _check_fastai_fastcore_pyproject_versions(storage_folder)328 329    from fastai.learner import load_learner  # type: ignore330 331    return load_learner(os.path.join(storage_folder, "model.pkl"))332 333 334@validate_hf_hub_args335def push_to_hub_fastai(336    learner,337    *,338    repo_id: str,339    commit_message: str = "Push FastAI model using huggingface_hub.",340    private: bool | None = None,341    token: str | None = None,342    config: dict | None = None,343    branch: str | None = None,344    create_pr: bool | None = None,345    allow_patterns: list[str] | str | None = None,346    ignore_patterns: list[str] | str | None = None,347    delete_patterns: list[str] | str | None = None,348    api_endpoint: str | None = None,349):350    """351    Upload learner checkpoint files to the Hub.352 353    Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use354    `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more355    details.356 357    Args:358        learner (`Learner`):359            The `fastai.Learner' you'd like to push to the Hub.360        repo_id (`str`):361            The repository id for your model in Hub in the format of "namespace/repo_name". The namespace can be your individual account or an organization to which you have write access (for example, 'stanfordnlp/stanza-de').362        commit_message (`str`, *optional*):363            Message to commit while pushing. Will default to :obj:`"add model"`.364        private (`bool`, *optional*):365            Whether or not the repository created should be private.366            If `None` (default), will default to been public except if the organization's default is private.367        token (`str`, *optional*):368            The Hugging Face account token to use as HTTP bearer authorization for remote files. If :obj:`None`, the token will be asked by a prompt.369        config (`dict`, *optional*):370            Configuration object to be saved alongside the model weights.371        branch (`str`, *optional*):372            The git branch on which to push the model. This defaults to373            the default branch as specified in your repository, which374            defaults to `"main"`.375        create_pr (`boolean`, *optional*):376            Whether or not to create a Pull Request from `branch` with that commit.377            Defaults to `False`.378        api_endpoint (`str`, *optional*):379            The API endpoint to use when pushing the model to the hub.380        allow_patterns (`list[str]` or `str`, *optional*):381            If provided, only files matching at least one pattern are pushed.382        ignore_patterns (`list[str]` or `str`, *optional*):383            If provided, files matching any of the patterns are not pushed.384        delete_patterns (`list[str]` or `str`, *optional*):385            If provided, remote files matching any of the patterns will be deleted from the repo.386 387    Returns:388        The url of the commit of your model in the given repository.389 390    > [!TIP]391    > Raises the following error:392    >393    >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)394    >       if the user is not log on to the Hugging Face Hub.395    """396    _check_fastai_fastcore_versions()397    api = HfApi(endpoint=api_endpoint)398    repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id399 400    # Push the files to the repo in a single commit401    with SoftTemporaryDirectory() as tmp:402        saved_path = Path(tmp) / repo_id403        _save_pretrained_fastai(learner, saved_path, config=config)404        return api.upload_folder(405            repo_id=repo_id,406            token=token,407            folder_path=saved_path,408            commit_message=commit_message,409            revision=branch,410            create_pr=create_pr,411            allow_patterns=allow_patterns,412            ignore_patterns=ignore_patterns,413            delete_patterns=delete_patterns,414        )415