codekingpro/portable-devtools

1# Copyright 202-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Contains command to download files from the Hub with the CLI.15 16Usage:17    hf download --help18 19    # Download file20    hf download gpt2 config.json21 22    # Download entire repo23    hf download fffiloni/zeroscope --repo-type=space --revision=refs/pr/7824 25    # Download repo with filters26    hf download gpt2 --include="*.safetensors"27 28    # Download with token29    hf download Wauplin/private-model --token=hf_***30 31    # Download quietly (no progress bar, no warnings, only the returned path)32    hf download gpt2 config.json --quiet33 34    # Download to local dir35    hf download gpt2 --local-dir=./models/gpt236 37    # Download a subfolder38    hf download HuggingFaceM4/FineVision art/ --repo-type=dataset39"""40 41import warnings42from typing import Annotated43 44import typer45 46from huggingface_hub._snapshot_download import snapshot_download47from huggingface_hub.errors import CLIError48from huggingface_hub.file_download import DryRunFileInfo, hf_hub_download49from huggingface_hub.utils import _format_size50 51from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt52from ._output import out53 54 55DOWNLOAD_EXAMPLES = [56    "hf download meta-llama/Llama-3.2-1B-Instruct",57    "hf download meta-llama/Llama-3.2-1B-Instruct config.json tokenizer.json",58    'hf download meta-llama/Llama-3.2-1B-Instruct --include "*.safetensors" --exclude "*.bin"',59    "hf download meta-llama/Llama-3.2-1B-Instruct --local-dir ./models/llama",60    "hf download HuggingFaceM4/FineVision art/ --repo-type dataset",61]62 63 64def download(65    repo_id: RepoIdArg,66    filenames: Annotated[67        list[str] | None,68        typer.Argument(69            help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",70        ),71    ] = None,72    repo_type: RepoTypeOpt = RepoTypeOpt.model,73    revision: RevisionOpt = None,74    include: Annotated[75        list[str] | None,76        typer.Option(77            help="Glob patterns to include from files to download. eg: *.json",78        ),79    ] = None,80    exclude: Annotated[81        list[str] | None,82        typer.Option(83            help="Glob patterns to exclude from files to download.",84        ),85    ] = None,86    cache_dir: Annotated[87        str | None,88        typer.Option(89            help="Directory where to save files.",90        ),91    ] = None,92    local_dir: Annotated[93        str | None,94        typer.Option(95            help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-a-local-folder for more details.",96        ),97    ] = None,98    force_download: Annotated[99        bool,100        typer.Option(101            help="If True, the files will be downloaded even if they are already cached.",102        ),103    ] = False,104    dry_run: Annotated[105        bool,106        typer.Option(107            help="If True, perform a dry run without actually downloading the file.",108        ),109    ] = False,110    token: TokenOpt = None,111    max_workers: Annotated[112        int,113        typer.Option(114            help="Maximum number of workers to use for downloading files. Default is 8.",115        ),116    ] = 8,117) -> None:118    """Download files from the Hub."""119 120    def run_download() -> str | DryRunFileInfo | list[DryRunFileInfo]:121        filenames_list = filenames if filenames is not None else []122 123        # Separate subfolder patterns (ending with '/') from regular filenames124        # Subfolders like "art/" are converted to include patterns like "art/**"125        subfolders = [f for f in filenames_list if f.endswith("/")]126        subfolder_patterns = [f"{f.rstrip('/')}/**" for f in subfolders]127        regular_filenames = [f for f in filenames_list if not f.endswith("/")]128 129        # Error if subfolder patterns are combined with --include/--exclude130        # Guide user to use --include instead of subfolder argument131        if len(subfolder_patterns) > 0:132            if include is not None and len(include) > 0:133                raise CLIError(134                    f"Cannot combine subfolder argument ('{subfolders[0]}') with `--include`. "135                    f'Please use `--include "{subfolders[0]}*"` instead.'136                )137            if exclude is not None and len(exclude) > 0:138                raise CLIError(139                    f"Cannot combine subfolder argument ('{subfolders[0]}') with `--exclude`. "140                    f'Please use `--include "{subfolders[0]}*"` with `--exclude` instead.'141                )142 143        # Warn user if patterns are ignored (only if regular filenames are provided)144        if len(regular_filenames) > 0:145            if include is not None and len(include) > 0:146                warnings.warn("Ignoring `--include` since filenames have being explicitly set.")147            if exclude is not None and len(exclude) > 0:148                warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")149 150        # Single file to download (not a subfolder): use `hf_hub_download`151        if len(regular_filenames) == 1 and len(subfolder_patterns) == 0:152            return hf_hub_download(153                repo_id=repo_id,154                repo_type=repo_type.value,155                revision=revision,156                filename=regular_filenames[0],157                cache_dir=cache_dir,158                force_download=force_download,159                token=token,160                local_dir=local_dir,161                library_name="huggingface-cli",162                dry_run=dry_run,163            )164 165        # Otherwise: use `snapshot_download` to ensure all files comes from same revision166        if len(regular_filenames) == 0 and len(subfolder_patterns) == 0:167            # No filenames provided: use include/exclude patterns168            allow_patterns = include169            ignore_patterns = exclude170        else:171            # Combine regular filenames and subfolder patterns as allow_patterns172            allow_patterns = regular_filenames + subfolder_patterns173            ignore_patterns = None174 175        return snapshot_download(176            repo_id=repo_id,177            repo_type=repo_type.value,178            revision=revision,179            allow_patterns=allow_patterns,180            ignore_patterns=ignore_patterns,181            force_download=force_download,182            cache_dir=cache_dir,183            token=token,184            local_dir=local_dir,185            library_name="huggingface-cli",186            max_workers=max_workers,187            dry_run=dry_run,188        )189 190    def _print_result(result: str | DryRunFileInfo | list[DryRunFileInfo]) -> None:191        if isinstance(result, str):192            out.result("Downloaded", path=result)193            return194 195        # Print dry run info196        if isinstance(result, DryRunFileInfo):197            result = [result]198        will_download = [r for r in result if r.will_download]199        out.text(200            f"[dry-run] Will download {len(will_download)} files"201            f" (out of {len(result)})"202            f" totalling {_format_size(sum(r.file_size for r in will_download))}."203        )204        items = [205            {206                "file": info.filename,207                "size": _format_size(info.file_size) if info.will_download else "-",208            }209            for info in sorted(result, key=lambda x: x.filename)210        ]211        out.table(items)212 213    _print_result(run_download())214