codekingpro/portable-devtools

1# Copyright 2023-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Contains command to upload a large folder with the CLI."""15 16import os17from typing import Annotated18 19import typer20 21from huggingface_hub import logging22from huggingface_hub.utils import disable_progress_bars23 24from ._cli_utils import (25    PrivateOpt,26    RepoIdArg,27    RepoType,28    RepoTypeOpt,29    RevisionOpt,30    TokenOpt,31    get_hf_api,32)33from ._output import out34 35 36logger = logging.get_logger(__name__)37 38 39UPLOAD_LARGE_FOLDER_EXAMPLES = [40    "hf upload-large-folder Wauplin/my-cool-model ./large_model_dir",41    "hf upload-large-folder Wauplin/my-cool-model ./large_model_dir --revision v1.0",42]43 44 45def upload_large_folder(46    repo_id: RepoIdArg,47    local_path: Annotated[48        str,49        typer.Argument(50            help="Local path to the folder to upload.",51        ),52    ],53    repo_type: RepoTypeOpt = RepoType.model,54    revision: RevisionOpt = None,55    private: PrivateOpt = None,56    include: Annotated[57        list[str] | None,58        typer.Option(59            help="Glob patterns to match files to upload.",60        ),61    ] = None,62    exclude: Annotated[63        list[str] | None,64        typer.Option(65            help="Glob patterns to exclude from files to upload.",66        ),67    ] = None,68    token: TokenOpt = None,69    num_workers: Annotated[70        int | None,71        typer.Option(72            help="Number of workers to use to hash, upload and commit files.",73        ),74    ] = None,75    no_report: Annotated[76        bool,77        typer.Option(78            help="Whether to disable regular status report.",79        ),80    ] = False,81    no_bars: Annotated[82        bool,83        typer.Option(84            help="Whether to disable progress bars.",85        ),86    ] = False,87) -> None:88    """Upload a large folder to the Hub. Recommended for resumable uploads."""89    if not os.path.isdir(local_path):90        raise typer.BadParameter("Large upload is only supported for folders.", param_hint="local_path")91 92    out.warning(93        "You are about to upload a large folder to the Hub using `hf upload-large-folder`. "94        "This is a new feature so feedback is very welcome!\n"95        "\n"96        "A few things to keep in mind:\n"97        "  - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"98        "  - Do not start several processes in parallel.\n"99        "  - You can interrupt and resume the process at any time. "100        "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"101        "  - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"102        "\n"103        f"Some temporary metadata will be stored under `{local_path}/.cache/huggingface`.\n"104        "  - You must not modify those files manually.\n"105        "  - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"106        "  - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"107        "\n"108        "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "109        "You can also entirely disable the status report with `--no-report`.\n"110        "\n"111        "For more details, run `hf upload-large-folder --help` or check the documentation at "112        "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."113    )114 115    if no_bars:116        disable_progress_bars()117 118    api = get_hf_api(token=token)119    api.upload_large_folder(120        repo_id=repo_id,121        folder_path=local_path,122        repo_type=repo_type.value,123        revision=revision,124        private=private,125        allow_patterns=include,126        ignore_patterns=exclude,127        num_workers=num_workers,128        print_report=not no_report,129    )130