codekingpro/portable-devtools
114k
1# Copyright 2019-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14from __future__ import annotations15 16import inspect17import itertools18import json19import re20import struct21import time22import warnings23from collections import defaultdict24from collections.abc import Callable, Iterable, Iterator25from concurrent.futures import Future, ThreadPoolExecutor26from dataclasses import asdict, dataclass, field27from datetime import datetime, timezone28from functools import wraps29from itertools import islice30from pathlib import Path31from secrets import token_hex32from typing import TYPE_CHECKING, Any, BinaryIO, Literal, TypeVar, overload33from urllib.parse import quote, unquote34 35import httpcore36import httpx37from tqdm.auto import tqdm as base_tqdm38from tqdm.contrib.concurrent import thread_map39 40from huggingface_hub.utils._xet import (41 XetTokenType,42 fetch_xet_connection_info_from_repo_info,43 reset_xet_connection_info_cache_for_repo,44)45 46from . import constants47from ._buckets import (48 BucketFile,49 BucketFileMetadata,50 BucketFolder,51 BucketInfo,52 BucketUrl,53 SyncPlan,54 _BucketAddFile,55 _BucketCopyFile,56 _BucketDeleteFile,57 _split_bucket_id_and_prefix,58 sync_bucket_internal,59)60from ._commit_api import (61 CommitOperation,62 CommitOperationAdd,63 CommitOperationCopy,64 CommitOperationDelete,65 _fetch_files_to_copy,66 _fetch_upload_modes,67 _prepare_commit_payload,68 _upload_files,69 _warn_on_overwriting_operations,70)71from ._dataset_viewer import DatasetParquetEntry72from ._eval_results import EvalResultEntry, parse_eval_result_entries73from ._inference_endpoints import InferenceEndpoint, InferenceEndpointScalingMetric, InferenceEndpointType74from ._jobs_api import JobHardware, JobInfo, JobSpec, ScheduledJobInfo, _create_job_spec75from ._space_api import (76 SpaceHardware,77 SpaceRuntime,78 SpaceSearchResult,79 SpaceSecret,80 SpaceStorage,81 SpaceVariable,82 Volume,83)84from ._upload_large_folder import upload_large_folder_internal85from .community import (86 Discussion,87 DiscussionComment,88 DiscussionStatusChange,89 DiscussionTitleChange,90 DiscussionWithDetails,91 deserialize_event,92)93from .errors import (94 BadRequestError,95 EntryNotFoundError,96 GatedRepoError,97 HfHubHTTPError,98 LocalTokenNotFoundError,99 RemoteEntryNotFoundError,100 RepositoryNotFoundError,101 RevisionNotFoundError,102 XetAuthorizationError,103 XetRefreshTokenError,104)105from .file_download import DryRunFileInfo, HfFileMetadata, get_hf_file_metadata, hf_hub_url106from .repocard_data import DatasetCardData, ModelCardData, SpaceCardData107from .utils import (108 DEFAULT_IGNORE_PATTERNS,109 NotASafetensorsRepoError,110 SafetensorsFileMetadata,111 SafetensorsParsingError,112 SafetensorsRepoMetadata,113 TensorInfo,114 are_progress_bars_disabled,115 build_hf_headers,116 chunk_iterable,117 experimental,118 filter_repo_objects,119 fix_hf_endpoint_in_url,120 get_session,121 get_token,122 hf_raise_for_status,123 http_backoff,124 logging,125 paginate,126 parse_datetime,127 parse_xet_file_data_from_response,128 refresh_xet_connection_info,129 silent_tqdm,130 validate_hf_hub_args,131)132from .utils import tqdm as hf_tqdm133from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab134from .utils._deprecation import _deprecate_arguments, _deprecate_method135from .utils._http import _httpx_follow_relative_redirects_with_backoff136from .utils._typing import CallableT137from .utils._verification import collect_local_files, resolve_local_root, verify_maps138from .utils.endpoint_helpers import _is_emission_within_threshold139from .utils.tqdm import _get_progress_bar_context140 141 142if TYPE_CHECKING:143 from .inference._providers import PROVIDER_T144 from .utils._verification import FolderVerification145 from .utils._xet_progress_reporting import XetProgressReporter146 147R = TypeVar("R") # Return type148CollectionItemType_T = Literal["model", "dataset", "space", "paper", "collection", "bucket"]149CollectionSort_T = Literal["lastModified", "trending", "upvotes"]150RepoVisibility_T = Literal["public", "private", "protected"]151 152ExpandModelProperty_T = Literal[153 "author",154 "baseModels",155 "cardData",156 "childrenModelCount",157 "config",158 "createdAt",159 "disabled",160 "downloads",161 "downloadsAllTime",162 "evalResults",163 "gated",164 "gguf",165 "inference",166 "inferenceProviderMapping",167 "lastModified",168 "library_name",169 "likes",170 "mask_token",171 "model-index",172 "pipeline_tag",173 "private",174 "resourceGroup",175 "safetensors",176 "sha",177 "siblings",178 "spaces",179 "tags",180 "transformersInfo",181 "trendingScore",182 "usedStorage",183 "widgetData",184]185 186ExpandDatasetProperty_T = Literal[187 "author",188 "cardData",189 "citation",190 "createdAt",191 "description",192 "disabled",193 "downloads",194 "downloadsAllTime",195 "gated",196 "lastModified",197 "likes",198 "mainSize",199 "paperswithcode_id",200 "private",201 "resourceGroup",202 "sha",203 "siblings",204 "tags",205 "trendingScore",206 "usedStorage",207]208 209ExpandSpaceProperty_T = Literal[210 "author",211 "cardData",212 "createdAt",213 "datasets",214 "disabled",215 "lastModified",216 "likes",217 "models",218 "private",219 "resourceGroup",220 "runtime",221 "sdk",222 "sha",223 "siblings",224 "subdomain",225 "tags",226 "trendingScore",227 "usedStorage",228]229 230ModelSort_T = Literal["created_at", "downloads", "last_modified", "likes", "trending_score"]231DatasetSort_T = Literal["created_at", "downloads", "last_modified", "likes", "trending_score"]232SpaceSort_T = Literal["created_at", "last_modified", "likes", "trending_score"]233DailyPapersSort_T = Literal["publishedAt", "trending"]234 235USERNAME_PLACEHOLDER = "hf_user"236_REGEX_DISCUSSION_URL = re.compile(r".*/discussions/(\d+)$")237_REGEX_HTTP_PROTOCOL = re.compile(r"https?://")238 239_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE = (240 "\nNote: Creating a commit assumes that the repo already exists on the"241 " Huggingface Hub. Please use `create_repo` if it's not the case."242)243_AUTH_CHECK_NO_REPO_ERROR_MESSAGE = (244 "\nNote: The repository either does not exist or you do not have access rights."245 " Please check the repository ID and your access permissions."246 " If this is a private repository, ensure that your token is correct."247)248_BUCKET_PATHS_INFO_BATCH_SIZE = 1000249_BUCKET_BATCH_ADD_CHUNK_SIZE = 1000250_BUCKET_BATCH_DELETE_CHUNK_SIZE = 1000251 252# Regex used to match special revisions with "/" in them (see #1710)253SPECIAL_REFS_REVISION_REGEX = re.compile(254 r"""255 (^refs\/convert\/\w+) # `refs/convert/parquet` revisions256 |257 (^refs\/pr\/\d+) # PR revisions258 """,259 re.VERBOSE,260)261 262logger = logging.get_logger(__name__)263 264 265def _resolve_repo_visibility(266 *,267 private: bool | None,268 visibility: RepoVisibility_T | None,269 repo_type: str | None,270) -> RepoVisibility_T | None:271 if private is not None and visibility is not None:272 raise ValueError("Received both `private` and `visibility` arguments. Please provide only one of them.")273 274 if visibility is None:275 if private is None:276 return None277 return "private" if private else "public"278 279 if visibility == "protected" and repo_type != constants.REPO_TYPE_SPACE:280 raise ValueError("Only Spaces can be 'protected'. Please set visibility to 'public' or 'private'.")281 return visibility282 283 284def repo_type_and_id_from_hf_id(hf_id: str, hub_url: str | None = None) -> tuple[str | None, str | None, str]:285 """286 Returns the repo type and ID from a huggingface.co URL linking to a287 repository288 289 Args:290 hf_id (`str`):291 An URL or ID of a repository on the HF hub. Accepted values are:292 293 - https://huggingface.co/<repo_type>/<namespace>/<repo_id>294 - https://huggingface.co/<namespace>/<repo_id>295 - hf://<repo_type>/<namespace>/<repo_id>296 - hf://<namespace>/<repo_id>297 - <repo_type>/<namespace>/<repo_id>298 - <namespace>/<repo_id>299 - <repo_id>300 hub_url (`str`, *optional*):301 The URL of the HuggingFace Hub, defaults to https://huggingface.co302 303 Returns:304 A tuple with three items: repo_type (`str` or `None`), namespace (`str` or305 `None`) and repo_id (`str`).306 307 Raises:308 [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)309 If URL cannot be parsed.310 [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)311 If `repo_type` is unknown.312 """313 input_hf_id = hf_id314 315 # Get the hub_url (with or without protocol)316 full_hub_url = hub_url if hub_url is not None else constants.ENDPOINT317 hub_url_without_protocol = _REGEX_HTTP_PROTOCOL.sub("", full_hub_url)318 319 # Check if hf_id is a URL containing the hub_url (check both with and without protocol)320 hf_id_without_protocol = _REGEX_HTTP_PROTOCOL.sub("", hf_id)321 is_hf_url = hub_url_without_protocol in hf_id_without_protocol and "@" not in hf_id322 323 HFFS_PREFIX = "hf://"324 if hf_id.startswith(HFFS_PREFIX): # Remove "hf://" prefix if exists325 hf_id = hf_id[len(HFFS_PREFIX) :]326 327 # If it's a URL, strip the endpoint prefix to get the path328 if is_hf_url:329 # Remove protocol if present330 hf_id_normalized = _REGEX_HTTP_PROTOCOL.sub("", hf_id)331 332 # Remove the hub_url prefix to get the relative path333 if hf_id_normalized.startswith(hub_url_without_protocol):334 # Strip the hub URL and any leading slashes335 hf_id = hf_id_normalized[len(hub_url_without_protocol) :].lstrip("/")336 337 url_segments = hf_id.split("/")338 is_hf_id = len(url_segments) <= 3339 340 namespace: str | None341 if is_hf_url:342 # For URLs, we need to extract repo_type, namespace, repo_id343 # Expected format after stripping endpoint: [repo_type]/namespace/repo_id or namespace/repo_id344 345 if len(url_segments) >= 3:346 # Check if first segment is a repo type347 if url_segments[0] in constants.REPO_TYPES_MAPPING:348 repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]]349 namespace = url_segments[1]350 repo_id = url_segments[2]351 elif url_segments[0] == "buckets":352 # Special case for buckets353 repo_type = "bucket"354 namespace = url_segments[1]355 repo_id = url_segments[2]356 else:357 # First segment is namespace358 namespace = url_segments[0]359 repo_id = url_segments[1]360 repo_type = None361 elif len(url_segments) == 2:362 namespace = url_segments[0]363 repo_id = url_segments[1]364 365 # Check if namespace is actually a repo type mapping366 if namespace in constants.REPO_TYPES_MAPPING:367 # Mean canonical dataset or model368 repo_type = constants.REPO_TYPES_MAPPING[namespace]369 namespace = None370 elif namespace == "buckets":371 # Special case for buckets372 repo_type = "bucket"373 namespace = None374 else:375 repo_type = None376 else:377 # Single segment378 repo_id = url_segments[0]379 namespace = None380 repo_type = None381 elif is_hf_id:382 if len(url_segments) == 3:383 # Passed <repo_type>/<user>/<model_id> or <repo_type>/<org>/<model_id>384 repo_type, namespace, repo_id = url_segments[-3:]385 elif len(url_segments) == 2:386 if url_segments[0] in constants.REPO_TYPES_MAPPING:387 # Passed '<model_id>' or 'datasets/<dataset_id>' for a canonical model or dataset388 repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]]389 namespace = None390 repo_id = hf_id.split("/")[-1]391 elif url_segments[0] == "buckets":392 # Special case for buckets393 repo_type = "bucket"394 namespace = None395 repo_id = hf_id.split("/")[-1]396 else:397 # Passed <user>/<model_id> or <org>/<model_id>398 namespace, repo_id = hf_id.split("/")[-2:]399 repo_type = None400 else:401 # Passed <model_id>402 repo_id = url_segments[0]403 namespace, repo_type = None, None404 else:405 raise ValueError(f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}")406 407 # Check if repo type is known (mapping "spaces" => "space" + empty value => `None`)408 if repo_type in constants.REPO_TYPES_MAPPING:409 repo_type = constants.REPO_TYPES_MAPPING[repo_type]410 if repo_type == "":411 repo_type = None412 if repo_type not in constants.REPO_TYPES_WITH_KERNEL and repo_type != "bucket":413 raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')")414 415 return repo_type, namespace, repo_id416 417 418def _parse_hf_copy_handle(hf_handle: str) -> _BucketCopyHandle | _RepoCopyHandle:419 # TODO: Harmonize hf:// parsing. See https://github.com/huggingface/huggingface_hub/issues/3971420 if not hf_handle.startswith("hf://"):421 raise ValueError(f"Invalid HF handle: '{hf_handle}'. Expected a path starting with 'hf://'.")422 423 path = hf_handle.removeprefix("hf://")424 if path.startswith("buckets/"):425 bucket_id, bucket_path = _split_bucket_id_and_prefix(path.removeprefix("buckets/"))426 return _BucketCopyHandle(427 bucket_id=bucket_id,428 path=bucket_path.strip("/"),429 )430 431 path = path.strip("/")432 if path == "":433 raise ValueError(f"Invalid HF handle: '{hf_handle}'.")434 435 parts = path.split("/")436 repo_type: str = constants.REPO_TYPE_MODEL437 if parts[0] in constants.REPO_TYPES_MAPPING:438 repo_type = constants.REPO_TYPES_MAPPING[parts[0]]439 parts = parts[1:]440 441 if len(parts) < 2:442 raise ValueError(443 f"Invalid repo HF handle: '{hf_handle}'. Expected format 'hf://<namespace>/<repo_id>/path' or with explicit repo type prefix."444 )445 446 namespace, repo_name_with_revision = parts[0], parts[1]447 remaining_parts = parts[2:]448 revision: str | None = None449 if "@" in repo_name_with_revision:450 repo_name, revision = repo_name_with_revision.split("@", 1)451 else:452 repo_name = repo_name_with_revision453 454 if revision is None:455 revision = constants.DEFAULT_REVISION456 else:457 revision = unquote(revision)458 if remaining_parts:459 maybe_special_ref = f"{revision}/{remaining_parts[0]}"460 match = SPECIAL_REFS_REVISION_REGEX.match(maybe_special_ref)461 if match is not None:462 revision = match.group()463 suffix = maybe_special_ref.removeprefix(revision).lstrip("/")464 remaining_parts = ([suffix] if suffix else []) + remaining_parts[1:]465 466 repo_path = "/".join(remaining_parts).strip("/")467 return _RepoCopyHandle(468 repo_type=repo_type, # type: ignore469 repo_id=f"{namespace}/{repo_name}",470 revision=revision,471 path=repo_path,472 )473 474 475@dataclass476class LastCommitInfo(dict):477 oid: str478 title: str479 date: datetime480 481 def __post_init__(self): # hack to make LastCommitInfo backward compatible482 self.update(asdict(self))483 484 485@dataclass486class BlobLfsInfo(dict):487 size: int488 sha256: str489 pointer_size: int490 491 def __post_init__(self): # hack to make BlobLfsInfo backward compatible492 self.update(asdict(self))493 494 495@dataclass496class BlobSecurityInfo(dict):497 safe: bool # duplicate information with "status" field, keeping it for backward compatibility498 status: str499 av_scan: dict | None500 pickle_import_scan: dict | None501 502 def __post_init__(self): # hack to make BlogSecurityInfo backward compatible503 self.update(asdict(self))504 505 506@dataclass507class TransformersInfo(dict):508 auto_model: str509 custom_class: str | None = None510 # possible `pipeline_tag` values: https://github.com/huggingface/huggingface.js/blob/3ee32554b8620644a6287e786b2a83bf5caf559c/packages/tasks/src/pipelines.ts#L72511 pipeline_tag: str | None = None512 processor: str | None = None513 514 def __post_init__(self): # hack to make TransformersInfo backward compatible515 self.update(asdict(self))516 517 518@dataclass519class SafeTensorsInfo(dict):520 parameters: dict[str, int]521 total: int522 523 def __post_init__(self): # hack to make SafeTensorsInfo backward compatible524 self.update(asdict(self))525 526 527@dataclass528class CommitInfo(str):529 """Data structure containing information about a newly created commit.530 531 Returned by any method that creates a commit on the Hub: [`create_commit`], [`upload_file`], [`upload_folder`],532 [`delete_file`], [`delete_folder`]. It inherits from `str` for backward compatibility but using methods specific533 to `str` is deprecated.534 535 Attributes:536 commit_url (`str`):537 Url where to find the commit.538 539 commit_message (`str`):540 The summary (first line) of the commit that has been created.541 542 commit_description (`str`):543 Description of the commit that has been created. Can be empty.544 545 oid (`str`):546 Commit hash id. Example: `"91c54ad1727ee830252e457677f467be0bfd8a57"`.547 548 pr_url (`str`, *optional*):549 Url to the PR that has been created, if any. Populated when `create_pr=True`550 is passed.551 552 pr_revision (`str`, *optional*):553 Revision of the PR that has been created, if any. Populated when554 `create_pr=True` is passed. Example: `"refs/pr/1"`.555 556 pr_num (`int`, *optional*):557 Number of the PR discussion that has been created, if any. Populated when558 `create_pr=True` is passed. Can be passed as `discussion_num` in559 [`get_discussion_details`]. Example: `1`.560 561 repo_url (`RepoUrl`):562 Repo URL of the commit containing info like repo_id, repo_type, etc.563 """564 565 commit_url: str566 commit_message: str567 commit_description: str568 oid: str569 _endpoint: str | None = field(default=None, repr=False)570 pr_url: str | None = None571 572 # Computed from `commit_url` in `__post_init__`573 repo_url: RepoUrl = field(init=False)574 575 # Computed from `pr_url` in `__post_init__`576 pr_revision: str | None = field(init=False)577 pr_num: int | None = field(init=False)578 579 def __new__(cls, *args, commit_url: str, **kwargs):580 return str.__new__(cls, commit_url)581 582 def __post_init__(self):583 """Populate pr-related fields after initialization.584 585 See https://docs.python.org/3.10/library/dataclasses.html#post-init-processing.586 """587 # Repo info588 self.repo_url = RepoUrl(self.commit_url.split("/commit/")[0], endpoint=self._endpoint)589 590 # PR info591 if self.pr_url is not None:592 self.pr_revision = _parse_revision_from_pr_url(self.pr_url)593 self.pr_num = int(self.pr_revision.split("/")[-1])594 else:595 self.pr_revision = None596 self.pr_num = None597 598 599@dataclass600class AccessRequest:601 """Data structure containing information about a user access request.602 603 Attributes:604 username (`str`):605 Username of the user who requested access.606 fullname (`str`):607 Fullname of the user who requested access.608 email (`Optional[str]`):609 Email of the user who requested access.610 Can only be `None` in the /accepted list if the user was granted access manually.611 timestamp (`datetime`):612 Timestamp of the request.613 status (`Literal["pending", "accepted", "rejected"]`):614 Status of the request. Can be one of `["pending", "accepted", "rejected"]`.615 fields (`dict[str, Any]`, *optional*):616 Additional fields filled by the user in the gate form.617 """618 619 username: str620 fullname: str621 email: str | None622 timestamp: datetime623 status: Literal["pending", "accepted", "rejected"]624 625 # Additional fields filled by the user in the gate form626 fields: dict[str, Any] | None = None627 628 629@dataclass630class WebhookWatchedItem:631 """Data structure containing information about the items watched by a webhook.632 633 Attributes:634 type (`Literal["dataset", "model", "org", "space", "user"]`):635 Type of the item to be watched. Can be one of `["dataset", "model", "org", "space", "user"]`.636 name (`str`):637 Name of the item to be watched. Can be the username, organization name, model name, dataset name or space name.638 """639 640 type: Literal["dataset", "model", "org", "space", "user"]641 name: str642 643 644@dataclass645class WebhookInfo:646 """Data structure containing information about a webhook.647 648 One of `url` or `job` is specified, but not both.649 650 Attributes:651 id (`str`):652 ID of the webhook.653 url (`str`, *optional*):654 URL of the webhook.655 job (`JobSpec`, *optional*):656 Specifications of the Job to trigger.657 watched (`list[WebhookWatchedItem]`):658 List of items watched by the webhook, see [`WebhookWatchedItem`].659 domains (`list[WEBHOOK_DOMAIN_T]`):660 List of domains the webhook is watching. Can be one of `["repo", "discussions"]`.661 secret (`str`, *optional*):662 Secret of the webhook.663 disabled (`bool`):664 Whether the webhook is disabled or not.665 """666 667 id: str668 url: str | None669 job: JobSpec | None670 watched: list[WebhookWatchedItem]671 domains: list[constants.WEBHOOK_DOMAIN_T]672 secret: str | None673 disabled: bool674 675 676class RepoUrl(str):677 """Subclass of `str` describing a repo URL on the Hub.678 679 `RepoUrl` is returned by `HfApi.create_repo`. It inherits from `str` for backward680 compatibility. At initialization, the URL is parsed to populate properties:681 - endpoint (`str`)682 - namespace (`Optional[str]`)683 - repo_name (`str`)684 - repo_id (`str`)685 - repo_type (`Literal["model", "dataset", "space"]`)686 - url (`str`)687 688 Args:689 url (`Any`):690 String value of the repo url.691 endpoint (`str`, *optional*):692 Endpoint of the Hub. Defaults to <https://huggingface.co>.693 694 Example:695 ```py696 >>> RepoUrl('https://huggingface.co/gpt2')697 RepoUrl('https://huggingface.co/gpt2', endpoint='https://huggingface.co', repo_type='model', repo_id='gpt2')698 699 >>> RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co')700 RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset')701 702 >>> RepoUrl('hf://datasets/my-user/my-dataset')703 RepoUrl('hf://datasets/my-user/my-dataset', endpoint='https://huggingface.co', repo_type='dataset', repo_id='user/dataset')704 705 >>> HfApi.create_repo("dummy_model")706 RepoUrl('https://huggingface.co/Wauplin/dummy_model', endpoint='https://huggingface.co', repo_type='model', repo_id='Wauplin/dummy_model')707 ```708 709 Raises:710 [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)711 If URL cannot be parsed.712 [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)713 If `repo_type` is unknown.714 """715 716 def __new__(cls, url: Any, endpoint: str | None = None):717 url = fix_hf_endpoint_in_url(url, endpoint=endpoint)718 return super().__new__(cls, url)719 720 def __init__(self, url: Any, endpoint: str | None = None) -> None:721 super().__init__()722 # Parse URL723 self.endpoint = endpoint or constants.ENDPOINT724 repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(self, hub_url=self.endpoint)725 726 # Populate fields727 self.namespace = namespace728 self.repo_name = repo_name729 self.repo_id = repo_name if namespace is None else f"{namespace}/{repo_name}"730 self.repo_type = repo_type or constants.REPO_TYPE_MODEL731 self.url = str(self) # just in case it's needed732 733 def __repr__(self) -> str:734 return f"RepoUrl('{self}', endpoint='{self.endpoint}', repo_type='{self.repo_type}', repo_id='{self.repo_id}')"735 736 737@dataclass(frozen=True)738class _BucketCopyHandle:739 bucket_id: str740 path: str741 742 743@dataclass(frozen=True)744class _RepoCopyHandle:745 repo_type: Literal["model", "dataset", "space"]746 repo_id: str747 revision: str748 path: str749 750 751@dataclass752class RepoSibling:753 """754 Contains basic information about a repo file inside a repo on the Hub.755 756 > [!TIP]757 > All attributes of this class are optional except `rfilename`. This is because only the file names are returned when758 > listing repositories on the Hub (with [`list_models`], [`list_datasets`] or [`list_spaces`]). If you need more759 > information like file size, blob id or lfs details, you must request them specifically from one repo at a time760 > (using [`model_info`], [`dataset_info`] or [`space_info`]) as it adds more constraints on the backend server to761 > retrieve these.762 763 Attributes:764 rfilename (str):765 file name, relative to the repo root.766 size (`int`, *optional*):767 The file's size, in bytes. This attribute is defined when `files_metadata` argument of [`repo_info`] is set768 to `True`. It's `None` otherwise.769 blob_id (`str`, *optional*):770 The file's git OID. This attribute is defined when `files_metadata` argument of [`repo_info`] is set to771 `True`. It's `None` otherwise.772 lfs (`BlobLfsInfo`, *optional*):773 The file's LFS metadata. This attribute is defined when`files_metadata` argument of [`repo_info`] is set to774 `True` and the file is stored with Git LFS. It's `None` otherwise.775 """776 777 rfilename: str778 size: int | None = None779 blob_id: str | None = None780 lfs: BlobLfsInfo | None = None781 782 783@dataclass784class RepoFile:785 """786 Contains information about a file on the Hub.787 788 Attributes:789 path (str):790 file path relative to the repo root.791 size (`int`):792 The file's size, in bytes.793 blob_id (`str`):794 The file's git OID.795 lfs (`BlobLfsInfo`, *optional*):796 The file's LFS metadata.797 xet_hash (`str`, *optional*):798 The file's Xet hash.799 last_commit (`LastCommitInfo`, *optional*):800 The file's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`]801 are called with `expand=True`.802 security (`BlobSecurityInfo`, *optional*):803 The file's security scan metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`]804 are called with `expand=True`.805 """806 807 path: str808 size: int809 blob_id: str810 lfs: BlobLfsInfo | None = None811 xet_hash: str | None = None812 last_commit: LastCommitInfo | None = None813 security: BlobSecurityInfo | None = None814 815 def __init__(self, **kwargs):816 self.path = kwargs.pop("path")817 self.size = kwargs.pop("size")818 self.blob_id = kwargs.pop("oid")819 lfs = kwargs.pop("lfs", None)820 if lfs is not None:821 lfs = BlobLfsInfo(size=lfs["size"], sha256=lfs["oid"], pointer_size=lfs["pointerSize"])822 self.lfs = lfs823 self.xet_hash = kwargs.pop("xetHash", None)824 last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None)825 if last_commit is not None:826 last_commit = LastCommitInfo(827 oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"])828 )829 self.last_commit = last_commit830 security = kwargs.pop("securityFileStatus", None)831 if security is not None:832 safe = security["status"] == "safe"833 security = BlobSecurityInfo(834 safe=safe,835 status=security["status"],836 av_scan=security["avScan"],837 pickle_import_scan=security["pickleImportScan"],838 )839 self.security = security840 841 # backwards compatibility842 self.rfilename = self.path843 self.lastCommit = self.last_commit844 845 846@dataclass847class RepoFolder:848 """849 Contains information about a folder on the Hub.850 851 Attributes:852 path (str):853 folder path relative to the repo root.854 tree_id (`str`):855 The folder's git OID.856 last_commit (`LastCommitInfo`, *optional*):857 The folder's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`]858 are called with `expand=True`.859 """860 861 path: str862 tree_id: str863 last_commit: LastCommitInfo | None = None864 865 def __init__(self, **kwargs):866 self.path = kwargs.pop("path")867 self.tree_id = kwargs.pop("oid")868 last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None)869 if last_commit is not None:870 last_commit = LastCommitInfo(871 oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"])872 )873 self.last_commit = last_commit874 875 876@dataclass877class InferenceProviderMapping:878 provider: PROVIDER_T # Provider name879 hf_model_id: str # ID of the model on the Hugging Face Hub880 provider_id: str # ID of the model on the provider's side881 status: Literal["error", "live", "staging"]882 task: str883 884 adapter: str | None = None885 adapter_weights_path: str | None = None886 type: Literal["single-model", "tag-filter"] | None = None887 888 def __init__(self, **kwargs):889 self.provider = kwargs.pop("provider")890 self.hf_model_id = kwargs.pop("hf_model_id")891 self.provider_id = kwargs.pop("providerId")892 self.status = kwargs.pop("status")893 self.task = kwargs.pop("task")894 895 self.adapter = kwargs.pop("adapter", None)896 self.adapter_weights_path = kwargs.pop("adapterWeightsPath", None)897 self.type = kwargs.pop("type", None)898 self.__dict__.update(**kwargs)899 900 901@dataclass902class ModelInfo:903 """904 Contains information about a model on the Hub. This object is returned by [`model_info`] and [`list_models`].905 906 > [!TIP]907 > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made.908 > In general, the more specific the query, the more information is returned. On the contrary, when listing models909 > using [`list_models`] only a subset of the attributes are returned.910 911 Attributes:912 id (`str`):913 ID of model.914 author (`str`, *optional*):915 Author of the model.916 base_models (`list[str]`, *optional*):917 List of base models this model is derived from.918 card_data (`ModelCardData`, *optional*):919 Model Card Metadata as a [`huggingface_hub.repocard_data.ModelCardData`] object.920 children_model_count (`int`, *optional*):921 Number of children models derived from this model.922 config (`dict`, *optional*):923 Model configuration.924 created_at (`datetime`, *optional*):925 Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`,926 corresponding to the date when we began to store creation dates.927 disabled (`bool`, *optional*):928 Is the repo disabled.929 downloads (`int`):930 Number of downloads of the model over the last 30 days.931 downloads_all_time (`int`):932 Cumulated number of downloads of the model since its creation.933 eval_results (`list[EvalResultEntry]`, *optional*):934 Model's evaluation results.935 gated (`Literal["auto", "manual", False]`, *optional*):936 Is the repo gated.937 If so, whether there is manual or automatic approval.938 gguf (`dict`, *optional*):939 GGUF information of the model.940 inference (`Literal["warm"]`, *optional*):941 Status of the model on Inference Providers. Warm if the model is served by at least one provider.942 inference_provider_mapping (`list[InferenceProviderMapping]`, *optional*):943 A list of [`InferenceProviderMapping`] ordered after the user's provider order.944 last_modified (`datetime`, *optional*):945 Date of last commit to the repo.946 library_name (`str`, *optional*):947 Library associated with the model.948 likes (`int`):949 Number of likes of the model.950 mask_token (`str`, *optional*):951 Mask token used by the model.952 model_index (`dict`, *optional*):953 Model index for evaluation.954 pipeline_tag (`str`, *optional*):955 Pipeline tag associated with the model.956 private (`bool`):957 Is the repo private.958 resource_group (`dict`, *optional*):959 Resource group information for the model.960 safetensors (`SafeTensorsInfo`, *optional*):961 Model's safetensors information.962 security_repo_status (`dict`, *optional*):963 Model's security scan status.964 sha (`str`, *optional*):965 Repo SHA at this particular revision.966 siblings (`list[RepoSibling]`):967 List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the model.968 spaces (`list[str]`, *optional*):969 List of spaces using the model.970 tags (`list[str]`):971 List of tags of the model. Compared to `card_data.tags`, contains extra tags computed by the Hub972 (e.g. supported libraries, model's arXiv).973 transformers_info (`TransformersInfo`, *optional*):974 Transformers-specific info (auto class, processor, etc.) associated with the model.975 trending_score (`int`, *optional*):976 Trending score of the model.977 used_storage (`int`, *optional*):978 Size in bytes of the model on the Hub.979 widget_data (`Any`, *optional*):980 Widget data associated with the model.981 """982 983 id: str984 author: str | None985 base_models: list[str] | None986 card_data: ModelCardData | None987 children_model_count: int | None988 config: dict | None989 created_at: datetime | None990 disabled: bool | None991 downloads: int | None992 downloads_all_time: int | None993 eval_results: list[EvalResultEntry] | None994 gated: Literal["auto", "manual", False] | None995 gguf: dict | None996 inference: Literal["warm"] | None997 inference_provider_mapping: list[InferenceProviderMapping] | None998 last_modified: datetime | None999 library_name: str | None1000 likes: int | None1001 mask_token: str | None1002 model_index: dict | None1003 pipeline_tag: str | None1004 private: bool | None1005 resource_group: dict | None1006 safetensors: SafeTensorsInfo | None1007 security_repo_status: dict | None1008 sha: str | None1009 siblings: list[RepoSibling] | None1010 spaces: list[str] | None1011 tags: list[str] | None1012 transformers_info: TransformersInfo | None1013 trending_score: int | None1014 used_storage: int | None1015 widget_data: Any | None1016 1017 def __init__(self, **kwargs):1018 self.id = kwargs.pop("id")1019 self.author = kwargs.pop("author", None)1020 self.sha = kwargs.pop("sha", None)1021 last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None)1022 self.last_modified = parse_datetime(last_modified) if last_modified else None1023 created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None)1024 self.created_at = parse_datetime(created_at) if created_at else None1025 self.private = kwargs.pop("private", None)1026 self.gated = kwargs.pop("gated", None)1027 self.disabled = kwargs.pop("disabled", None)1028 self.downloads = kwargs.pop("downloads", None)1029 self.downloads_all_time = kwargs.pop("downloadsAllTime", None)1030 self.likes = kwargs.pop("likes", None)1031 self.library_name = kwargs.pop("library_name", None)1032 self.gguf = kwargs.pop("gguf", None)1033 1034 self.inference = kwargs.pop("inference", None)1035 1036 # little hack to simplify Inference Providers logic and make it backward and forward compatible1037 # right now, API returns a dict on model_info and a list on list_models. Let's harmonize to list.1038 mapping = kwargs.pop("inferenceProviderMapping", None)1039 if isinstance(mapping, list):1040 self.inference_provider_mapping = [1041 InferenceProviderMapping(**{**value, "hf_model_id": self.id}) for value in mapping1042 ]1043 elif isinstance(mapping, dict):1044 self.inference_provider_mapping = [1045 InferenceProviderMapping(**{**value, "hf_model_id": self.id, "provider": provider})1046 for provider, value in mapping.items()1047 ]1048 elif mapping is None:1049 self.inference_provider_mapping = None1050 else:1051 raise ValueError(1052 f"Unexpected type for `inferenceProviderMapping`. Expecting `dict` or `list`. Got {mapping}."1053 )1054 1055 self.tags = kwargs.pop("tags", None)1056 self.pipeline_tag = kwargs.pop("pipeline_tag", None)1057 self.mask_token = kwargs.pop("mask_token", None)1058 self.trending_score = kwargs.pop("trendingScore", None)1059 self.used_storage = kwargs.pop("usedStorage", None)1060 1061 card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None)1062 self.card_data = (1063 ModelCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data1064 )1065 1066 self.widget_data = kwargs.pop("widgetData", None)1067 self.model_index = kwargs.pop("model-index", None) or kwargs.pop("model_index", None)1068 self.config = kwargs.pop("config", None)1069 transformers_info = kwargs.pop("transformersInfo", None) or kwargs.pop("transformers_info", None)1070 self.transformers_info = TransformersInfo(**transformers_info) if transformers_info else None1071 siblings = kwargs.pop("siblings", None)1072 self.siblings = (1073 [1074 RepoSibling(1075 rfilename=sibling["rfilename"],1076 size=sibling.get("size"),1077 blob_id=sibling.get("blobId"),1078 lfs=(1079 BlobLfsInfo(1080 size=sibling["lfs"]["size"],1081 sha256=sibling["lfs"]["sha256"],1082 pointer_size=sibling["lfs"]["pointerSize"],1083 )1084 if sibling.get("lfs")1085 else None1086 ),1087 )1088 for sibling in siblings1089 ]1090 if siblings is not None1091 else None1092 )1093 self.spaces = kwargs.pop("spaces", None)1094 safetensors = kwargs.pop("safetensors", None)1095 self.safetensors = (1096 SafeTensorsInfo(1097 parameters=safetensors["parameters"],1098 total=safetensors["total"],1099 )1100 if safetensors1101 else None1102 )1103 self.security_repo_status = kwargs.pop("securityRepoStatus", None)1104 eval_results = kwargs.pop("evalResults", None)1105 self.eval_results = parse_eval_result_entries(eval_results) if eval_results else None1106 self.base_models = kwargs.pop("baseModels", None)1107 self.children_model_count = kwargs.pop("childrenModelCount", None)1108 self.resource_group = kwargs.pop("resourceGroup", None)1109 # backwards compatibility1110 self.lastModified = self.last_modified1111 self.cardData = self.card_data1112 self.transformersInfo = self.transformers_info1113 self.__dict__.update(**kwargs)1114 1115 1116@dataclass1117class DatasetInfo:1118 """1119 Contains information about a dataset on the Hub. This object is returned by [`dataset_info`] and [`list_datasets`].1120 1121 > [!TIP]1122 > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made.1123 > In general, the more specific the query, the more information is returned. On the contrary, when listing datasets1124 > using [`list_datasets`] only a subset of the attributes are returned.1125 1126 Attributes:1127 id (`str`):1128 ID of dataset.1129 author (`str`):1130 Author of the dataset.1131 card_data (`DatasetCardData`, *optional*):1132 Dataset Card Metadata as a [`huggingface_hub.repocard_data.DatasetCardData`] object.1133 citation (`str`, *optional*):1134 Citation information for the dataset.1135 created_at (`datetime`, *optional*):1136 Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`,1137 corresponding to the date when we began to store creation dates.1138 description (`str`, *optional*):1139 Description of the dataset.1140 disabled (`bool`, *optional*):1141 Is the repo disabled.1142 downloads (`int`):1143 Number of downloads of the dataset over the last 30 days.1144 downloads_all_time (`int`):1145 Cumulated number of downloads of the dataset since its creation.1146 gated (`Literal["auto", "manual", False]`, *optional*):1147 Is the repo gated.1148 If so, whether there is manual or automatic approval.1149 last_modified (`datetime`, *optional*):1150 Date of last commit to the repo.1151 likes (`int`):1152 Number of likes of the dataset.1153 main_size (`int`, *optional*):1154 Size in bytes of the main branch of the dataset.1155 paperswithcode_id (`str`, *optional*):1156 Papers with code ID of the dataset.1157 private (`bool`):1158 Is the repo private.1159 resource_group (`dict`, *optional*):1160 Resource group information for the dataset.1161 sha (`str`):1162 Repo SHA at this particular revision.1163 siblings (`list[RepoSibling]`):1164 List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the dataset.1165 tags (`list[str]`):1166 List of tags of the dataset.1167 trending_score (`int`, *optional*):1168 Trending score of the dataset.1169 used_storage (`int`, *optional*):1170 Size in bytes of the dataset on the Hub.1171 """1172 1173 id: str1174 author: str | None1175 card_data: DatasetCardData | None1176 citation: str | None1177 created_at: datetime | None1178 description: str | None1179 disabled: bool | None1180 downloads: int | None1181 downloads_all_time: int | None1182 gated: Literal["auto", "manual", False] | None1183 last_modified: datetime | None1184 likes: int | None1185 main_size: int | None1186 paperswithcode_id: str | None1187 private: bool | None1188 resource_group: dict | None1189 sha: str | None1190 siblings: list[RepoSibling] | None1191 tags: list[str] | None1192 trending_score: int | None1193 used_storage: int | None1194 1195 def __init__(self, **kwargs):1196 self.id = kwargs.pop("id")1197 self.author = kwargs.pop("author", None)1198 self.sha = kwargs.pop("sha", None)1199 created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None)1200 self.created_at = parse_datetime(created_at) if created_at else None