codekingpro/portable-devtools

1"""2Type definitions and utilities for the `create_commit` API3"""4 5import base646import io7import os8import warnings9from collections import defaultdict10from collections.abc import Iterable, Iterator11from contextlib import contextmanager12from dataclasses import dataclass, field13from itertools import groupby14from pathlib import Path, PurePosixPath15from typing import TYPE_CHECKING, Any, BinaryIO, Literal, Union16 17from tqdm.contrib.concurrent import thread_map18 19from . import constants20from .errors import EntryNotFoundError, HfHubHTTPError, XetAuthorizationError, XetRefreshTokenError21from .file_download import hf_hub_url22from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info23from .utils import (24    FORBIDDEN_FOLDERS,25    XetTokenType,26    are_progress_bars_disabled,27    chunk_iterable,28    fetch_xet_connection_info_from_repo_info,29    get_session,30    hf_raise_for_status,31    http_backoff,32    logging,33    sha,34    tqdm_stream_file,35    validate_hf_hub_args,36)37from .utils import tqdm as hf_tqdm38from .utils._runtime import is_xet_available39 40 41if TYPE_CHECKING:42    from .hf_api import RepoFile43 44 45logger = logging.get_logger(__name__)46 47 48UploadMode = Literal["lfs", "regular"]49 50# Max is 1,000 per request on the Hub for HfApi.get_paths_info51# Otherwise we get:52# HfHubHTTPError: 413 Client Error: Payload Too Large for url: https://huggingface.co/api/datasets/xxx (Request ID: xxx)\n\ntoo many parameters53# See https://github.com/huggingface/huggingface_hub/issues/150354FETCH_LFS_BATCH_SIZE = 50055 56UPLOAD_BATCH_MAX_NUM_FILES = 25657 58 59@dataclass60class CommitOperationDelete:61    """62    Data structure holding necessary info to delete a file or a folder from a repository63    on the Hub.64 65    Args:66        path_in_repo (`str`):67            Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"`68            for a file or `"checkpoints/1fec34a/"` for a folder.69        is_folder (`bool` or `Literal["auto"]`, *optional*)70            Whether the Delete Operation applies to a folder or not. If "auto", the path71            type (file or folder) is guessed automatically by looking if path ends with72            a "/" (folder) or not (file). To explicitly set the path type, you can set73            `is_folder=True` or `is_folder=False`.74    """75 76    path_in_repo: str77    is_folder: bool | Literal["auto"] = "auto"78 79    def __post_init__(self):80        self.path_in_repo = _validate_path_in_repo(self.path_in_repo)81 82        if self.is_folder == "auto":83            self.is_folder = self.path_in_repo.endswith("/")84        if not isinstance(self.is_folder, bool):85            raise ValueError(86                f"Wrong value for `is_folder`. Must be one of [`True`, `False`, `'auto'`]. Got '{self.is_folder}'."87            )88 89 90@dataclass91class CommitOperationCopy:92    """93    Data structure holding necessary info to copy a file in a repository on the Hub.94 95    Limitations:96      - Only LFS files can be copied. To copy a regular file, you need to download it locally and re-upload it97      - Cross-repository copies are not supported.98 99    Note: you can combine a [`CommitOperationCopy`] and a [`CommitOperationDelete`] to rename an LFS file on the Hub.100 101    Args:102        src_path_in_repo (`str`):103            Relative filepath in the repo of the file to be copied, e.g. `"checkpoints/1fec34a/weights.bin"`.104        path_in_repo (`str`):105            Relative filepath in the repo where to copy the file, e.g. `"checkpoints/1fec34a/weights_copy.bin"`.106        src_revision (`str`, *optional*):107            The git revision of the file to be copied. Can be any valid git revision.108            Default to the target commit revision.109    """110 111    src_path_in_repo: str112    path_in_repo: str113    src_revision: str | None = None114    # set to the OID of the file to be copied if it has already been uploaded115    # useful to determine if a commit will be empty or not.116    _src_oid: str | None = None117    # set to the OID of the file to copy to if it has already been uploaded118    # useful to determine if a commit will be empty or not.119    _dest_oid: str | None = None120 121    def __post_init__(self):122        self.src_path_in_repo = _validate_path_in_repo(self.src_path_in_repo)123        self.path_in_repo = _validate_path_in_repo(self.path_in_repo)124 125 126@dataclass127class CommitOperationAdd:128    """129    Data structure holding necessary info to upload a file to a repository on the Hub.130 131    Args:132        path_in_repo (`str`):133            Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"`134        path_or_fileobj (`str`, `Path`, `bytes`, or `BinaryIO`):135            Either:136            - a path to a local file (as `str` or `pathlib.Path`) to upload137            - a buffer of bytes (`bytes`) holding the content of the file to upload138            - a "file object" (subclass of `io.BufferedIOBase`), typically obtained139                with `open(path, "rb")`. It must support `seek()` and `tell()` methods.140 141    Raises:142        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)143            If `path_or_fileobj` is not one of `str`, `Path`, `bytes` or `io.BufferedIOBase`.144        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)145            If `path_or_fileobj` is a `str` or `Path` but not a path to an existing file.146        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)147            If `path_or_fileobj` is a `io.BufferedIOBase` but it doesn't support both148            `seek()` and `tell()`.149    """150 151    path_in_repo: str152    path_or_fileobj: str | Path | bytes | BinaryIO153    upload_info: UploadInfo = field(init=False, repr=False)154 155    # Internal attributes156 157    # set to "lfs" or "regular" once known158    _upload_mode: UploadMode | None = field(init=False, repr=False, default=None)159 160    # set to True if .gitignore rules prevent the file from being uploaded as LFS161    # (server-side check)162    _should_ignore: bool | None = field(init=False, repr=False, default=None)163 164    # set to the remote OID of the file if it has already been uploaded165    # useful to determine if a commit will be empty or not166    _remote_oid: str | None = field(init=False, repr=False, default=None)167 168    # set to True once the file has been uploaded as LFS169    _is_uploaded: bool = field(init=False, repr=False, default=False)170 171    # set to True once the file has been committed172    _is_committed: bool = field(init=False, repr=False, default=False)173 174    def __post_init__(self) -> None:175        """Validates `path_or_fileobj` and compute `upload_info`."""176        self.path_in_repo = _validate_path_in_repo(self.path_in_repo)177 178        # Validate `path_or_fileobj` value179        if isinstance(self.path_or_fileobj, Path):180            self.path_or_fileobj = str(self.path_or_fileobj)181        if isinstance(self.path_or_fileobj, str):182            path_or_fileobj = os.path.normpath(os.path.expanduser(self.path_or_fileobj))183            if not os.path.isfile(path_or_fileobj):184                raise ValueError(f"Provided path: '{path_or_fileobj}' is not a file on the local file system")185        elif not isinstance(self.path_or_fileobj, (io.BufferedIOBase, bytes)):186            # ^^ Inspired from: https://stackoverflow.com/questions/44584829/how-to-determine-if-file-is-opened-in-binary-or-text-mode187            raise ValueError(188                "path_or_fileobj must be either an instance of str, bytes or"189                " io.BufferedIOBase. If you passed a file-like object, make sure it is"190                " in binary mode."191            )192        if isinstance(self.path_or_fileobj, io.BufferedIOBase):193            try:194                self.path_or_fileobj.tell()195                self.path_or_fileobj.seek(0, os.SEEK_CUR)196            except (OSError, AttributeError) as exc:197                raise ValueError(198                    "path_or_fileobj is a file-like object but does not implement seek() and tell()"199                ) from exc200 201        # Compute "upload_info" attribute202        if isinstance(self.path_or_fileobj, str):203            self.upload_info = UploadInfo.from_path(self.path_or_fileobj)204        elif isinstance(self.path_or_fileobj, bytes):205            self.upload_info = UploadInfo.from_bytes(self.path_or_fileobj)206        else:207            self.upload_info = UploadInfo.from_fileobj(self.path_or_fileobj)208 209    @contextmanager210    def as_file(self, with_tqdm: bool = False) -> Iterator[BinaryIO]:211        """212        A context manager that yields a file-like object allowing to read the underlying213        data behind `path_or_fileobj`.214 215        Args:216            with_tqdm (`bool`, *optional*, defaults to `False`):217                If True, iterating over the file object will display a progress bar. Only218                works if the file-like object is a path to a file. Pure bytes and buffers219                are not supported.220 221        Example:222 223        ```python224        >>> operation = CommitOperationAdd(225        ...        path_in_repo="remote/dir/weights.h5",226        ...        path_or_fileobj="./local/weights.h5",227        ... )228        CommitOperationAdd(path_in_repo='remote/dir/weights.h5', path_or_fileobj='./local/weights.h5')229 230        >>> with operation.as_file() as file:231        ...     content = file.read()232 233        >>> with operation.as_file(with_tqdm=True) as file:234        ...     while True:235        ...         data = file.read(1024)236        ...         if not data:237        ...              break238        config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]239 240        >>> with operation.as_file(with_tqdm=True) as file:241        ...     httpx.put(..., data=file)242        config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]243        ```244        """245        if isinstance(self.path_or_fileobj, str) or isinstance(self.path_or_fileobj, Path):246            if with_tqdm:247                with tqdm_stream_file(self.path_or_fileobj) as file:248                    yield file249            else:250                with open(self.path_or_fileobj, "rb") as file:251                    yield file252        elif isinstance(self.path_or_fileobj, bytes):253            yield io.BytesIO(self.path_or_fileobj)254        elif isinstance(self.path_or_fileobj, io.BufferedIOBase):255            prev_pos = self.path_or_fileobj.tell()256            yield self.path_or_fileobj257            self.path_or_fileobj.seek(prev_pos, io.SEEK_SET)258 259    def b64content(self) -> bytes:260        """261        The base64-encoded content of `path_or_fileobj`262 263        Returns: `bytes`264        """265        with self.as_file() as file:266            return base64.b64encode(file.read())267 268    @property269    def _local_oid(self) -> str | None:270        """Return the OID of the local file.271 272        This OID is then compared to `self._remote_oid` to check if the file has changed compared to the remote one.273        If the file did not change, we won't upload it again to prevent empty commits.274 275        For LFS files, the OID corresponds to the SHA256 of the file content (used a LFS ref).276        For regular files, the OID corresponds to the SHA1 of the file content.277        Note: this is slightly different to git OID computation since the oid of an LFS file is usually the git-SHA1 of the278              pointer file content (not the actual file content). However, using the SHA256 is enough to detect changes279              and more convenient client-side.280        """281        if self._upload_mode is None:282            return None283        elif self._upload_mode == "lfs":284            return self.upload_info.sha256.hex()285        else:286            # Regular file => compute sha1287            # => no need to read by chunk since the file is guaranteed to be <=5MB.288            with self.as_file() as file:289                return sha.git_hash(file.read())290 291 292def _validate_path_in_repo(path_in_repo: str) -> str:293    # Validate `path_in_repo` value to prevent a server-side issue294    if path_in_repo.startswith("/"):295        path_in_repo = path_in_repo[1:]296    if path_in_repo == "." or path_in_repo == ".." or path_in_repo.startswith("../"):297        raise ValueError(f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'")298    if path_in_repo.startswith("./"):299        path_in_repo = path_in_repo[2:]300    for forbidden in FORBIDDEN_FOLDERS:301        if any(part == forbidden for part in path_in_repo.split("/")):302            raise ValueError(303                f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:"304                f" '{path_in_repo}')."305            )306    return path_in_repo307 308 309CommitOperation = Union[CommitOperationAdd, CommitOperationCopy, CommitOperationDelete]310 311 312def _warn_on_overwriting_operations(operations: list[CommitOperation]) -> None:313    """314    Warn user when a list of operations is expected to overwrite itself in a single315    commit.316 317    Rules:318    - If a filepath is updated by multiple `CommitOperationAdd` operations, a warning319      message is triggered.320    - If a filepath is updated at least once by a `CommitOperationAdd` and then deleted321      by a `CommitOperationDelete`, a warning is triggered.322    - If a `CommitOperationDelete` deletes a filepath that is then updated by a323      `CommitOperationAdd`, no warning is triggered. This is usually useless (no need to324      delete before upload) but can happen if a user deletes an entire folder and then325      add new files to it.326    """327    nb_additions_per_path: dict[str, int] = defaultdict(int)328    for operation in operations:329        path_in_repo = operation.path_in_repo330        if isinstance(operation, CommitOperationAdd):331            if nb_additions_per_path[path_in_repo] > 0:332                warnings.warn(333                    "About to update multiple times the same file in the same commit:"334                    f" '{path_in_repo}'. This can cause undesired inconsistencies in"335                    " your repo."336                )337            nb_additions_per_path[path_in_repo] += 1338            for parent in PurePosixPath(path_in_repo).parents:339                # Also keep track of number of updated files per folder340                # => warns if deleting a folder overwrite some contained files341                nb_additions_per_path[str(parent)] += 1342        if isinstance(operation, CommitOperationDelete):343            if nb_additions_per_path[str(PurePosixPath(path_in_repo))] > 0:344                if operation.is_folder:345                    warnings.warn(346                        "About to delete a folder containing files that have just been"347                        f" updated within the same commit: '{path_in_repo}'. This can"348                        " cause undesired inconsistencies in your repo."349                    )350                else:351                    warnings.warn(352                        "About to delete a file that have just been updated within the"353                        f" same commit: '{path_in_repo}'. This can cause undesired"354                        " inconsistencies in your repo."355                    )356 357 358@validate_hf_hub_args359def _upload_files(360    *,361    additions: list[CommitOperationAdd],362    repo_type: str,363    repo_id: str,364    headers: dict[str, str],365    endpoint: str | None = None,366    num_threads: int = 5,367    revision: str | None = None,368    create_pr: bool | None = None,369):370    """371    Negotiates per-file transfer (LFS vs Xet) and uploads in batches.372    """373    xet_additions: list[CommitOperationAdd] = []374    lfs_actions: list[dict[str, Any]] = []375    lfs_oid2addop: dict[str, CommitOperationAdd] = {}376 377    for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):378        chunk_list = [op for op in chunk]379 380        transfers: list[str] = ["basic", "multipart"]381        has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in chunk_list)382        if is_xet_available():383            if not has_buffered_io_data:384                transfers.append("xet")385            else:386                logger.warning(387                    "Uploading files as a binary IO buffer is not supported by Xet Storage. "388                    "Falling back to HTTP upload."389                )390 391        actions_chunk, errors_chunk, chosen_transfer = post_lfs_batch_info(392            upload_infos=[op.upload_info for op in chunk_list],393            repo_id=repo_id,394            repo_type=repo_type,395            revision=revision,396            endpoint=endpoint,397            headers=headers,398            token=None,  # already passed in 'headers'399            transfers=transfers,400        )401        if errors_chunk:402            message = "\n".join(403                [404                    f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"405                    for err in errors_chunk406                ]407            )408            raise ValueError(f"LFS batch API returned errors:\n{message}")409 410        # If server returns a transfer we didn't offer (e.g "xet" while uploading from BytesIO),411        # fall back to LFS for this chunk.412        if chosen_transfer == "xet" and ("xet" in transfers):413            xet_additions.extend(chunk_list)414        else:415            lfs_actions.extend(actions_chunk)416            for op in chunk_list:417                lfs_oid2addop[op.upload_info.sha256.hex()] = op418 419    if len(lfs_actions) > 0:420        _upload_lfs_files(421            actions=lfs_actions,422            oid2addop=lfs_oid2addop,423            headers=headers,424            endpoint=endpoint,425            num_threads=num_threads,426        )427 428    if len(xet_additions) > 0:429        _upload_xet_files(430            additions=xet_additions,431            repo_type=repo_type,432            repo_id=repo_id,433            headers=headers,434            endpoint=endpoint,435            revision=revision,436            create_pr=create_pr,437        )438 439 440@validate_hf_hub_args441def _upload_lfs_files(442    *,443    actions: list[dict[str, Any]],444    oid2addop: dict[str, CommitOperationAdd],445    headers: dict[str, str],446    endpoint: str | None = None,447    num_threads: int = 5,448):449    """450    Uploads the content of `additions` to the Hub using the large file storage protocol.451 452    Relevant external documentation:453        - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md454 455    Args:456        actions (`list[dict[str, Any]]`):457            LFS batch actions returned by the server.458        oid2addop (`dict[str, CommitOperationAdd]`):459            A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object.460        headers (`dict[str, str]`):461            Headers to use for the request, including authorization headers and user agent.462        endpoint (`str`, *optional*):463            The endpoint to use for the request. Defaults to `constants.ENDPOINT`.464        num_threads (`int`, *optional*):465            The number of concurrent threads to use when uploading. Defaults to 5.466 467    Raises:468        [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)469            If an upload failed for any reason470        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)471            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.472        repo_id (`str`):473            A namespace (user or an organization) and a repo name separated474            by a `/`.475        headers (`dict[str, str]`):476            Headers to use for the request, including authorization headers and user agent.477        num_threads (`int`, *optional*):478            The number of concurrent threads to use when uploading. Defaults to 5.479        revision (`str`, *optional*):480            The git revision to upload to.481 482    Raises:483        [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)484            If an upload failed for any reason485        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)486            If the server returns malformed responses487        [`HfHubHTTPError`]488            If the LFS batch endpoint returned an HTTP error.489    """490    # Filter out files already present upstream491    filtered_actions = []492    for action in actions:493        if action.get("actions") is None:494            logger.debug(495                f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload."496            )497        else:498            filtered_actions.append(action)499 500    # Upload according to server-provided actions501    def _wrapped_lfs_upload(batch_action) -> None:502        try:503            operation = oid2addop[batch_action["oid"]]504            lfs_upload(operation=operation, lfs_batch_action=batch_action, headers=headers, endpoint=endpoint)505        except Exception as exc:506            raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc507 508    if len(filtered_actions) == 1:509        logger.debug("Uploading 1 LFS file to the Hub")510        _wrapped_lfs_upload(filtered_actions[0])511    else:512        logger.debug(513            f"Uploading {len(filtered_actions)} LFS files to the Hub using up to {num_threads} threads concurrently"514        )515        thread_map(516            _wrapped_lfs_upload,517            filtered_actions,518            desc=f"Upload {len(filtered_actions)} LFS files",519            max_workers=num_threads,520            tqdm_class=hf_tqdm,521        )522 523 524@validate_hf_hub_args525def _upload_xet_files(526    *,527    additions: list[CommitOperationAdd],528    repo_type: str,529    repo_id: str,530    headers: dict[str, str],531    endpoint: str | None = None,532    revision: str | None = None,533    create_pr: bool | None = None,534):535    """536    Uploads the content of `additions` to the Hub using the xet storage protocol.537    This chunks the files and deduplicates the chunks before uploading them to xetcas storage.538 539    Args:540        additions (`` of `CommitOperationAdd`):541            The files to be uploaded.542        repo_type (`str`):543            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.544        repo_id (`str`):545            A namespace (user or an organization) and a repo name separated546            by a `/`.547        headers (`dict[str, str]`):548            Headers to use for the request, including authorization headers and user agent.549        endpoint: (`str`, *optional*):550            The endpoint to use for the xetcas service. Defaults to `constants.ENDPOINT`.551        revision (`str`, *optional*):552            The git revision to upload to.553        create_pr (`bool`, *optional*):554            Whether or not to create a Pull Request with that commit.555 556    Raises:557        [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)558            If an upload failed for any reason.559        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)560            If the server returns malformed responses or if the user is unauthorized to upload to xet storage.561        [`HfHubHTTPError`]562            If the LFS batch endpoint returned an HTTP error.563 564    **How it works:**565        The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks566            for efficient storage and transfer.567 568        `hf_xet.upload_files` manages uploading files by:569            - Taking a list of file paths to upload570            - Breaking files into smaller chunks for efficient storage571            - Avoiding duplicate storage by recognizing identical chunks across files572            - Connecting to a storage server (CAS server) that manages these chunks573 574        The upload process works like this:575        1. Create a local folder at ~/.cache/huggingface/xet/chunk-cache to store file chunks for reuse.576        2. Process files in parallel (up to 8 files at once):577            2.1. Read the file content.578            2.2. Split the file content into smaller chunks based on content patterns: each chunk gets a unique ID based on what's in it.579            2.3. For each chunk:580                - Check if it already exists in storage.581                - Skip uploading chunks that already exist.582            2.4. Group chunks into larger blocks for efficient transfer.583            2.5. Upload these blocks to the storage server.584            2.6. Create and upload information about how the file is structured.585        3. Return reference files that contain information about the uploaded files, which can be used later to download them.586    """587    if len(additions) == 0:588        return589 590    # at this point, we know that hf_xet is installed591    from hf_xet import upload_bytes, upload_files592 593    from .utils._xet_progress_reporting import XetProgressReporter594 595    try:596        xet_connection_info = fetch_xet_connection_info_from_repo_info(597            token_type=XetTokenType.WRITE,598            repo_id=repo_id,599            repo_type=repo_type,600            revision=revision,601            headers=headers,602            endpoint=endpoint,603            params={"create_pr": "1"} if create_pr else None,604        )605    except HfHubHTTPError as e:606        if e.response.status_code == 401:607            raise XetAuthorizationError(608                f"You are unauthorized to upload to xet storage for {repo_type}/{repo_id}. "609                f"Please check that you have configured your access token with write access to the repo."610            ) from e611        raise612 613    xet_endpoint = xet_connection_info.endpoint614    access_token_info = (xet_connection_info.access_token, xet_connection_info.expiration_unix_epoch)615 616    def token_refresher() -> tuple[str, int]:617        new_xet_connection = fetch_xet_connection_info_from_repo_info(618            token_type=XetTokenType.WRITE,619            repo_id=repo_id,620            repo_type=repo_type,621            revision=revision,622            headers=headers,623            endpoint=endpoint,624            params={"create_pr": "1"} if create_pr else None,625        )626        if new_xet_connection is None:627            raise XetRefreshTokenError("Failed to refresh xet token")628        return new_xet_connection.access_token, new_xet_connection.expiration_unix_epoch629 630    if not are_progress_bars_disabled():631        progress = XetProgressReporter()632        progress_callback = progress.update_progress633    else:634        progress, progress_callback = None, None635 636    try:637        all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)]638        all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))]639 640        xet_headers = headers.copy()641        xet_headers.pop("authorization", None)642 643        if len(all_paths_ops) > 0:644            all_paths = [str(op.path_or_fileobj) for op in all_paths_ops]645            all_sha256s = [op.upload_info.sha256.hex() for op in all_paths_ops]646            upload_files(647                all_paths,648                xet_endpoint,649                access_token_info,650                token_refresher,651                progress_callback,652                repo_type,653                request_headers=xet_headers,654                sha256s=all_sha256s,655            )656 657        if len(all_bytes_ops) > 0:658            all_bytes = [op.path_or_fileobj for op in all_bytes_ops]659            all_sha256s = [op.upload_info.sha256.hex() for op in all_bytes_ops]660            upload_bytes(661                all_bytes,662                xet_endpoint,663                access_token_info,664                token_refresher,665                progress_callback,666                repo_type,667                request_headers=xet_headers,668                sha256s=all_sha256s,669            )670 671    finally:672        if progress is not None:673            progress.close(False)674 675    return676 677 678def _validate_preupload_info(preupload_info: dict):679    files = preupload_info.get("files")680    if not isinstance(files, list):681        raise ValueError("preupload_info is improperly formatted")682    for file_info in files:683        if not (684            isinstance(file_info, dict)685            and isinstance(file_info.get("path"), str)686            and isinstance(file_info.get("uploadMode"), str)687            and (file_info["uploadMode"] in ("lfs", "regular"))688        ):689            raise ValueError("preupload_info is improperly formatted:")690    return preupload_info691 692 693@validate_hf_hub_args694def _fetch_upload_modes(695    additions: Iterable[CommitOperationAdd],696    repo_type: str,697    repo_id: str,698    headers: dict[str, str],699    revision: str,700    endpoint: str | None = None,701    create_pr: bool = False,702    gitignore_content: str | None = None,703) -> None:704    """705    Requests the Hub "preupload" endpoint to determine whether each input file should be uploaded as a regular git blob,706    as a git LFS blob, or as a XET file. Input `additions` are mutated in-place with the upload mode.707 708    Args:709        additions (`Iterable` of :class:`CommitOperationAdd`):710            Iterable of :class:`CommitOperationAdd` describing the files to711            upload to the Hub.712        repo_type (`str`):713            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.714        repo_id (`str`):715            A namespace (user or an organization) and a repo name separated716            by a `/`.717        headers (`dict[str, str]`):718            Headers to use for the request, including authorization headers and user agent.719        revision (`str`):720            The git revision to upload the files to. Can be any valid git revision.721        gitignore_content (`str`, *optional*):722            The content of the `.gitignore` file to know which files should be ignored. The order of priority723            is to first check if `gitignore_content` is passed, then check if the `.gitignore` file is present724            in the list of files to commit and finally default to the `.gitignore` file already hosted on the Hub725            (if any).726    Raises:727        [`~utils.HfHubHTTPError`]728            If the Hub API returned an error.729        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)730            If the Hub API response is improperly formatted.731    """732    endpoint = endpoint if endpoint is not None else constants.ENDPOINT733 734    # Fetch upload mode (LFS or regular) chunk by chunk.735    upload_modes: dict[str, UploadMode] = {}736    should_ignore_info: dict[str, bool] = {}737    oid_info: dict[str, str | None] = {}738 739    for chunk in chunk_iterable(additions, 256):740        payload: dict = {741            "files": [742                {743                    "path": op.path_in_repo,744                    "sample": base64.b64encode(op.upload_info.sample).decode("ascii"),745                    "size": op.upload_info.size,746                }747                for op in chunk748            ]749        }750        if gitignore_content is not None:751            payload["gitIgnore"] = gitignore_content752 753        resp = http_backoff(754            "POST",755            f"{endpoint}/api/{repo_type}s/{repo_id}/preupload/{revision}",756            json=payload,757            headers=headers,758            params={"create_pr": "1"} if create_pr else None,759        )760        hf_raise_for_status(resp)761        preupload_info = _validate_preupload_info(resp.json())762        upload_modes.update(**{file["path"]: file["uploadMode"] for file in preupload_info["files"]})763        should_ignore_info.update(**{file["path"]: file["shouldIgnore"] for file in preupload_info["files"]})764        oid_info.update(**{file["path"]: file.get("oid") for file in preupload_info["files"]})765 766    # Set upload mode for each addition operation767    for addition in additions:768        addition._upload_mode = upload_modes[addition.path_in_repo]769        addition._should_ignore = should_ignore_info[addition.path_in_repo]770        addition._remote_oid = oid_info[addition.path_in_repo]771 772    # Empty files cannot be uploaded as LFS (S3 would fail with a 501 Not Implemented)773    # => empty files are uploaded as "regular" to still allow users to commit them.774    for addition in additions:775        if addition.upload_info.size == 0:776            addition._upload_mode = "regular"777 778 779@validate_hf_hub_args780def _fetch_files_to_copy(781    copies: Iterable[CommitOperationCopy],782    repo_type: str,783    repo_id: str,784    headers: dict[str, str],785    revision: str,786    endpoint: str | None = None,787) -> dict[tuple[str, str | None], Union["RepoFile", bytes]]:788    """789    Fetch information about the files to copy.790 791    For LFS files, we only need their metadata (file size and sha256) while for regular files792    we need to download the raw content from the Hub.793 794    Args:795        copies (`Iterable` of :class:`CommitOperationCopy`):796            Iterable of :class:`CommitOperationCopy` describing the files to797            copy on the Hub.798        repo_type (`str`):799            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.800        repo_id (`str`):801            A namespace (user or an organization) and a repo name separated802            by a `/`.803        headers (`dict[str, str]`):804            Headers to use for the request, including authorization headers and user agent.805        revision (`str`):806            The git revision to upload the files to. Can be any valid git revision.807 808    Returns: `dict[tuple[str, Optional[str]], Union[RepoFile, bytes]]]`809        Key is the file path and revision of the file to copy.810        Value is the raw content as bytes (for regular files) or the file information as a RepoFile (for LFS files).811 812    Raises:813        [`~utils.HfHubHTTPError`]814            If the Hub API returned an error.815        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)816            If the Hub API response is improperly formatted.817    """818    from .hf_api import HfApi, RepoFolder819 820    hf_api = HfApi(endpoint=endpoint, headers=headers)821    files_to_copy: dict[tuple[str, str | None], Union["RepoFile", bytes]] = {}822    # Store (path, revision) -> oid mapping823    oid_info: dict[tuple[str, str | None], str | None] = {}824    # 1. Fetch OIDs for destination paths in batches.825    dest_paths = [op.path_in_repo for op in copies]826    for offset in range(0, len(dest_paths), FETCH_LFS_BATCH_SIZE):827        dest_repo_files = hf_api.get_paths_info(828            repo_id=repo_id,829            paths=dest_paths[offset : offset + FETCH_LFS_BATCH_SIZE],830            revision=revision,831            repo_type=repo_type,832        )833        for file in dest_repo_files:834            if not isinstance(file, RepoFolder):835                oid_info[(file.path, revision)] = file.blob_id836 837    # 2. Group by source revision and fetch source file info in batches.838    for src_revision, operations in groupby(copies, key=lambda op: op.src_revision):839        operations = list(operations)  # type: ignore840        src_paths = [op.src_path_in_repo for op in operations]841        for offset in range(0, len(src_paths), FETCH_LFS_BATCH_SIZE):842            src_repo_files = hf_api.get_paths_info(843                repo_id=repo_id,844                paths=src_paths[offset : offset + FETCH_LFS_BATCH_SIZE],845                revision=src_revision or revision,846                repo_type=repo_type,847            )848 849            for src_repo_file in src_repo_files:850                if isinstance(src_repo_file, RepoFolder):851                    raise NotImplementedError("Copying a folder is not implemented.")852                oid_info[(src_repo_file.path, src_revision)] = src_repo_file.blob_id853                # If it's an LFS file, store the RepoFile object. Otherwise, download raw bytes.854                if src_repo_file.lfs:855                    files_to_copy[(src_repo_file.path, src_revision)] = src_repo_file856                else:857                    # TODO: (optimization) download regular files to copy concurrently858                    url = hf_hub_url(859                        endpoint=endpoint,860                        repo_type=repo_type,861                        repo_id=repo_id,862                        revision=src_revision or revision,863                        filename=src_repo_file.path,864                    )865                    response = get_session().get(url, headers=headers)866                    hf_raise_for_status(response)867                    files_to_copy[(src_repo_file.path, src_revision)] = response.content868        # 3. Ensure all operations found a corresponding file in the Hub869        #  and track src/dest OIDs for each operation.870        for operation in operations:871            if (operation.src_path_in_repo, src_revision) not in files_to_copy:872                raise EntryNotFoundError(873                    f"Cannot copy {operation.src_path_in_repo} at revision "874                    f"{src_revision or revision}: file is missing on repo."875                )876            operation._src_oid = oid_info.get((operation.src_path_in_repo, operation.src_revision))877            operation._dest_oid = oid_info.get((operation.path_in_repo, revision))878    return files_to_copy879 880 881def _prepare_commit_payload(882    operations: Iterable[CommitOperation],883    files_to_copy: dict[tuple[str, str | None], Union["RepoFile", bytes]],884    commit_message: str,885    commit_description: str | None = None,886    parent_commit: str | None = None,887) -> Iterable[dict[str, Any]]:888    """889    Builds the payload to POST to the `/commit` API of the Hub.890 891    Payload is returned as an iterator so that it can be streamed as a ndjson in the892    POST request.893 894    For more information, see:895        - https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073896        - http://ndjson.org/897    """898    commit_description = commit_description if commit_description is not None else ""899 900    # 1. Send a header item with the commit metadata901    header_value = {"summary": commit_message, "description": commit_description}902    if parent_commit is not None:903        header_value["parentCommit"] = parent_commit904    yield {"key": "header", "value": header_value}905 906    nb_ignored_files = 0907 908    # 2. Send operations, one per line909    for operation in operations:910        # Skip ignored files911        if isinstance(operation, CommitOperationAdd) and operation._should_ignore:912            logger.debug(f"Skipping file '{operation.path_in_repo}' in commit (ignored by gitignore file).")913            nb_ignored_files += 1914            continue915 916        # 2.a. Case adding a regular file917        if isinstance(operation, CommitOperationAdd) and operation._upload_mode == "regular":918            yield {919                "key": "file",920                "value": {921                    "content": operation.b64content().decode(),922                    "path": operation.path_in_repo,923                    "encoding": "base64",924                },925            }926        # 2.b. Case adding an LFS file927        elif isinstance(operation, CommitOperationAdd) and operation._upload_mode == "lfs":928            yield {929                "key": "lfsFile",930                "value": {931                    "path": operation.path_in_repo,932                    "algo": "sha256",933                    "oid": operation.upload_info.sha256.hex(),934                    "size": operation.upload_info.size,935                },936            }937        # 2.c. Case deleting a file or folder938        elif isinstance(operation, CommitOperationDelete):939            yield {940                "key": "deletedFolder" if operation.is_folder else "deletedFile",941                "value": {"path": operation.path_in_repo},942            }943        # 2.d. Case copying a file or folder944        elif isinstance(operation, CommitOperationCopy):945            file_to_copy = files_to_copy[(operation.src_path_in_repo, operation.src_revision)]946            if isinstance(file_to_copy, bytes):947                yield {948                    "key": "file",949                    "value": {950                        "content": base64.b64encode(file_to_copy).decode(),951                        "path": operation.path_in_repo,952                        "encoding": "base64",953                    },954                }955            elif file_to_copy.lfs:956                yield {957                    "key": "lfsFile",958                    "value": {959                        "path": operation.path_in_repo,960                        "algo": "sha256",961                        "oid": file_to_copy.lfs.sha256,962                    },963                }964            else:965                raise ValueError(966                    "Malformed files_to_copy (should be raw file content as bytes or RepoFile objects with LFS info."967                )968        # 2.e. Never expected to happen969        else:970            raise ValueError(971                f"Unknown operation to commit. Operation: {operation}. Upload mode:"972                f" {getattr(operation, '_upload_mode', None)}"973            )974 975    if nb_ignored_files > 0:976        logger.info(f"Skipped {nb_ignored_files} file(s) in commit (ignored by gitignore file).")977