codekingpro/portable-devtools

1# Copyright 2022-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Contains utilities to handle paths in Huggingface Hub."""15 16from collections.abc import Callable, Generator, Iterable17from fnmatch import fnmatch18from pathlib import Path19from typing import TypeVar20 21 22T = TypeVar("T")23 24# Always ignore `.git` and `.cache/huggingface` folders in commits25DEFAULT_IGNORE_PATTERNS = [26    ".git",27    ".git/*",28    "*/.git",29    "**/.git/**",30    ".cache/huggingface",31    ".cache/huggingface/*",32    "*/.cache/huggingface",33    "**/.cache/huggingface/**",34]35# Forbidden to commit these folders36FORBIDDEN_FOLDERS = [".git", ".cache"]37 38 39def filter_repo_objects(40    items: Iterable[T],41    *,42    allow_patterns: list[str] | str | None = None,43    ignore_patterns: list[str] | str | None = None,44    key: Callable[[T], str] | None = None,45) -> Generator[T, None, None]:46    """Filter repo objects based on an allowlist and a denylist.47 48    Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects.49    In the later case, `key` must be provided and specifies a function of one argument50    that is used to extract a path from each element in iterable.51 52    Patterns are Standard Wildcards (globbing patterns), NOT regular expressions.53    The pattern matching is based on Python's `fnmatch`. Note that `fnmatch` matches54    `*` across path boundaries, unlike traditional Unix shell globbing. For example,55    `"data/*.json"` will match both `data/file.json` and `data/subdir/file.json`.56    See https://docs.python.org/3/library/fnmatch.html for more details.57 58    Args:59        items (`Iterable`):60            List of items to filter.61        allow_patterns (`str` or `list[str]`, *optional*):62            Patterns constituting the allowlist. If provided, item paths must match at63            least one pattern from the allowlist.64        ignore_patterns (`str` or `list[str]`, *optional*):65            Patterns constituting the denylist. If provided, item paths must not match66            any patterns from the denylist.67        key (`Callable[[T], str]`, *optional*):68            Single-argument function to extract a path from each item. If not provided,69            the `items` must already be `str` or `Path`.70 71    Returns:72        Filtered list of objects, as a generator.73 74    Raises:75        :class:`ValueError`:76            If `key` is not provided and items are not `str` or `Path`.77 78    Example usage with paths:79    ```python80    >>> # Filter only PDFs that are not hidden.81    >>> list(filter_repo_objects(82    ...     ["aaa.PDF", "bbb.jpg", ".ccc.pdf", ".ddd.png"],83    ...     allow_patterns=["*.pdf"],84    ...     ignore_patterns=[".*"],85    ... ))86    ["aaa.pdf"]87    ```88 89    Example usage with objects:90    ```python91    >>> list(filter_repo_objects(92    ... [93    ...     CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")94    ...     CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg")95    ...     CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf")96    ...     CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png")97    ... ],98    ... allow_patterns=["*.pdf"],99    ... ignore_patterns=[".*"],100    ... key=lambda x: x.repo_in_path101    ... ))102    [CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")]103    ```104    """105    if isinstance(allow_patterns, str):106        allow_patterns = [allow_patterns]107 108    if isinstance(ignore_patterns, str):109        ignore_patterns = [ignore_patterns]110 111    if allow_patterns is not None:112        allow_patterns = [_add_wildcard_to_directories(p) for p in allow_patterns]113    if ignore_patterns is not None:114        ignore_patterns = [_add_wildcard_to_directories(p) for p in ignore_patterns]115 116    if key is None:117 118        def _identity(item: T) -> str:119            if isinstance(item, str):120                return item121            if isinstance(item, Path):122                return str(item)123            raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.")124 125        key = _identity  # Items must be `str` or `Path`, otherwise raise ValueError126 127    for item in items:128        path = key(item)129 130        # Skip if there's an allowlist and path doesn't match any131        if allow_patterns is not None and not any(fnmatch(path, r) for r in allow_patterns):132            continue133 134        # Skip if there's a denylist and path matches any135        if ignore_patterns is not None and any(fnmatch(path, r) for r in ignore_patterns):136            continue137 138        yield item139 140 141def _add_wildcard_to_directories(pattern: str) -> str:142    if pattern[-1] == "/":143        return pattern + "*"144    return pattern145