codekingpro/portable-devtools
114k
1import copy2import errno3import os4import re5import shutil6import stat7import time8import uuid9import warnings10from dataclasses import dataclass11from pathlib import Path12from typing import Any, BinaryIO, Literal, NoReturn, overload13from urllib.parse import quote, urlparse14 15import httpx16from tqdm.auto import tqdm as base_tqdm17 18from . import constants19from ._local_folder import (20 _create_cachedir_tag,21 get_local_download_paths,22 read_download_metadata,23 write_download_metadata,24)25from .errors import (26 FileMetadataError,27 GatedRepoError,28 HfHubHTTPError,29 LocalEntryNotFoundError,30 RemoteEntryNotFoundError,31 RepositoryNotFoundError,32 RevisionNotFoundError,33)34from .utils import (35 OfflineModeIsEnabled,36 SoftTemporaryDirectory,37 WeakFileLock,38 XetFileData,39 build_hf_headers,40 hf_raise_for_status,41 logging,42 parse_xet_file_data_from_response,43 refresh_xet_connection_info,44 tqdm,45 validate_hf_hub_args,46)47from .utils._http import (48 _DEFAULT_RETRY_ON_EXCEPTIONS,49 _DEFAULT_RETRY_ON_STATUS_CODES,50 _adjust_range_header,51 _httpx_follow_relative_redirects_with_backoff,52 http_stream_backoff,53)54from .utils._runtime import is_xet_available55from .utils.sha import sha_fileobj56from .utils.tqdm import _get_progress_bar_context57 58 59logger = logging.get_logger(__name__)60 61# Return value when trying to load a file from cache but the file does not exist in the distant repo.62_CACHED_NO_EXIST = object()63_CACHED_NO_EXIST_T = Any64 65# Regex to get filename from a "Content-Disposition" header for CDN-served files66HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P<filename>.*?)";')67 68# Regex to check if the revision IS directly a commit_hash69REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$")70 71# Regex to check if the file etag IS a valid sha25672REGEX_SHA256 = re.compile(r"^[0-9a-f]{64}$")73 74_are_symlinks_supported_in_dir: dict[str, bool] = {}75 76# Internal retry timeout for metadata fetch when no local file exists77_ETAG_RETRY_TIMEOUT = 6078 79 80def are_symlinks_supported(cache_dir: str | Path | None = None) -> bool:81 """Return whether the symlinks are supported on the machine.82 83 Since symlinks support can change depending on the mounted disk, we need to check84 on the precise cache folder. By default, the default HF cache directory is checked.85 86 Args:87 cache_dir (`str`, `Path`, *optional*):88 Path to the folder where cached files are stored.89 90 Returns: [bool] Whether symlinks are supported in the directory.91 """92 # Defaults to HF cache93 if cache_dir is None:94 cache_dir = constants.HF_HUB_CACHE95 cache_dir = str(Path(cache_dir).expanduser().resolve()) # make it unique96 97 # If symlinks are explicitly disabled by the user, always return False98 if constants.HF_HUB_DISABLE_SYMLINKS:99 return False100 101 # Check symlink compatibility only once (per cache directory) at first time use102 if cache_dir not in _are_symlinks_supported_in_dir:103 _are_symlinks_supported_in_dir[cache_dir] = True104 105 os.makedirs(cache_dir, exist_ok=True)106 with SoftTemporaryDirectory(dir=cache_dir) as tmpdir:107 src_path = Path(tmpdir) / "dummy_file_src"108 src_path.touch()109 dst_path = Path(tmpdir) / "dummy_file_dst"110 111 # Relative source path as in `_create_symlink``112 relative_src = os.path.relpath(src_path, start=os.path.dirname(dst_path))113 try:114 os.symlink(relative_src, dst_path)115 except OSError:116 # Likely running on Windows117 _are_symlinks_supported_in_dir[cache_dir] = False118 119 if not constants.HF_HUB_DISABLE_SYMLINKS_WARNING:120 message = (121 "`huggingface_hub` cache-system uses symlinks by default to"122 " efficiently store duplicated files but your machine does not"123 f" support them in {cache_dir}. Caching files will still work"124 " but in a degraded version that might require more space on"125 " your disk. This warning can be disabled by setting the"126 " `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For"127 " more details, see"128 " https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations."129 )130 if os.name == "nt":131 message += (132 "\nTo support symlinks on Windows, you either need to"133 " activate Developer Mode or to run Python as an"134 " administrator. In order to activate developer mode,"135 " see this article:"136 " https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development"137 )138 warnings.warn(message)139 140 return _are_symlinks_supported_in_dir[cache_dir]141 142 143@dataclass(frozen=True)144class HfFileMetadata:145 """Data structure containing information about a file versioned on the Hub.146 147 Returned by [`get_hf_file_metadata`] based on a URL.148 149 Args:150 commit_hash (`str`, *optional*):151 The commit_hash related to the file.152 etag (`str`, *optional*):153 Etag of the file on the server.154 location (`str`):155 Location where to download the file. Can be a Hub url or not (CDN).156 size (`size`):157 Size of the file. In case of an LFS file, contains the size of the actual158 LFS file, not the pointer.159 xet_file_data (`XetFileData`, *optional*):160 Xet information for the file. This is only set if the file is stored using Xet storage.161 """162 163 commit_hash: str | None164 etag: str | None165 location: str166 size: int | None167 xet_file_data: XetFileData | None168 169 170@dataclass171class DryRunFileInfo:172 """Information returned when performing a dry run of a file download.173 174 Returned by [`hf_hub_download`] when `dry_run=True`.175 176 Args:177 commit_hash (`str`):178 The commit_hash related to the file.179 file_size (`int`):180 Size of the file. In case of an LFS file, contains the size of the actual LFS file, not the pointer.181 filename (`str`):182 Name of the file in the repo.183 is_cached (`bool`):184 Whether the file is already cached locally.185 will_download (`bool`):186 Whether the file will be downloaded if `hf_hub_download` is called with `dry_run=False`.187 In practice, will_download is `True` if the file is not cached or if `force_download=True`.188 """189 190 commit_hash: str191 file_size: int192 filename: str193 local_path: str194 is_cached: bool195 will_download: bool196 197 198@validate_hf_hub_args199def hf_hub_url(200 repo_id: str,201 filename: str,202 *,203 subfolder: str | None = None,204 repo_type: str | None = None,205 revision: str | None = None,206 endpoint: str | None = None,207) -> str:208 """Construct the URL of a file from the given information.209 210 The resolved address can either be a huggingface.co-hosted url, or a link to211 Cloudfront (a Content Delivery Network, or CDN) for large files which are212 more than a few MBs.213 214 Args:215 repo_id (`str`):216 A namespace (user or an organization) name and a repo name separated217 by a `/`.218 filename (`str`):219 The name of the file in the repo.220 subfolder (`str`, *optional*):221 An optional value corresponding to a folder inside the repo.222 repo_type (`str`, *optional*):223 Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo,224 `None` or `"model"` if downloading from a model. Default is `None`.225 revision (`str`, *optional*):226 An optional Git revision id which can be a branch name, a tag, or a227 commit hash.228 229 Example:230 231 ```python232 >>> from huggingface_hub import hf_hub_url233 234 >>> hf_hub_url(235 ... repo_id="julien-c/EsperBERTo-small", filename="pytorch_model.bin"236 ... )237 'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin'238 ```239 240 > [!TIP]241 > Notes:242 >243 > Cloudfront is replicated over the globe so downloads are way faster for244 > the end user (and it also lowers our bandwidth costs).245 >246 > Cloudfront aggressively caches files by default (default TTL is 24247 > hours), however this is not an issue here because we implement a248 > git-based versioning system on huggingface.co, which means that we store249 > the files on S3/Cloudfront in a content-addressable way (i.e., the file250 > name is its hash). Using content-addressable filenames means cache can't251 > ever be stale.252 >253 > In terms of client-side caching from this library, we base our caching254 > on the objects' entity tag (`ETag`), which is an identifier of a255 > specific version of a resource [1]_. An object's ETag is: its git-sha1256 > if stored in git, or its sha256 if stored in git-lfs.257 258 References:259 260 - [1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag261 """262 if subfolder == "":263 subfolder = None264 if subfolder is not None:265 filename = f"{subfolder}/{filename}"266 267 if repo_type not in constants.REPO_TYPES_WITH_KERNEL:268 raise ValueError("Invalid repo type")269 270 if repo_type in constants.REPO_TYPES_URL_PREFIXES:271 repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id272 273 if revision is None:274 revision = constants.DEFAULT_REVISION275 url = constants.HUGGINGFACE_CO_URL_TEMPLATE.format(276 repo_id=repo_id, revision=quote(revision, safe=""), filename=quote(filename)277 )278 # Update endpoint if provided279 if endpoint is not None and url.startswith(constants.ENDPOINT):280 url = endpoint + url[len(constants.ENDPOINT) :]281 return url282 283 284def _get_file_length_from_http_response(response: httpx.Response) -> int | None:285 """286 Get the length of the file from the HTTP response headers.287 288 This function extracts the file size from the HTTP response headers, either from the289 `Content-Range` or `Content-Length` header, if available (in that order).290 291 Args:292 response (`httpx.Response`):293 The HTTP response object.294 295 Returns:296 `int` or `None`: The length of the file in bytes, or None if not available.297 """298 299 # If HTTP response contains compressed body (e.g. gzip), the `Content-Length` header will300 # contain the length of the compressed body, not the uncompressed file size.301 # And at the start of transmission there's no way to know the uncompressed file size for gzip,302 # thus we return None in that case.303 content_encoding = response.headers.get("Content-Encoding", "identity").lower()304 if content_encoding != "identity":305 # gzip/br/deflate/zstd etc306 return None307 308 content_range = response.headers.get("Content-Range")309 if content_range is not None:310 return int(content_range.rsplit("/")[-1])311 312 content_length = response.headers.get("Content-Length")313 if content_length is not None:314 return int(content_length)315 316 return None317 318 319@validate_hf_hub_args320def http_get(321 url: str,322 temp_file: BinaryIO,323 *,324 resume_size: int = 0,325 headers: dict[str, Any] | None = None,326 expected_size: int | None = None,327 displayed_filename: str | None = None,328 tqdm_class: type[base_tqdm] | None = None,329 _nb_retries: int = 5,330 _tqdm_bar: tqdm | None = None,331) -> None:332 """333 Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub.334 335 If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely a336 transient error (network outage?). We log a warning message and try to resume the download a few times before337 giving up. The method gives up after 5 attempts if no new data has being received from the server.338 339 Args:340 url (`str`):341 The URL of the file to download.342 temp_file (`BinaryIO`):343 The file-like object where to save the file.344 resume_size (`int`, *optional*):345 The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a346 positive number, the download will resume at the given position.347 headers (`dict`, *optional*):348 Dictionary of HTTP Headers to send with the request.349 expected_size (`int`, *optional*):350 The expected size of the file to download. If set, the download will raise an error if the size of the351 received content is different from the expected one.352 displayed_filename (`str`, *optional*):353 The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If354 not set, the filename is guessed from the URL or the `Content-Disposition` header.355 """356 if expected_size is not None and resume_size == expected_size:357 # If the file is already fully downloaded, we don't need to download it again.358 return359 360 initial_headers = headers361 headers = copy.deepcopy(headers) or {}362 if resume_size > 0:363 headers["Range"] = _adjust_range_header(headers.get("Range"), resume_size)364 elif expected_size and expected_size > constants.MAX_HTTP_DOWNLOAD_SIZE:365 # Any files over 50GB will not be available through basic http requests.366 raise ValueError(367 "The file is too large to be downloaded using the regular download method. "368 " Install `hf_xet` with `pip install hf_xet` for xet-powered downloads."369 )370 371 with http_stream_backoff(372 method="GET",373 url=url,374 headers=headers,375 timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,376 retry_on_exceptions=(),377 retry_on_status_codes=(429,),378 ) as response:379 hf_raise_for_status(response)380 381 # If we requested a Range but got 200 back, the server ignored our Range header382 # (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption.383 if resume_size > 0 and response.status_code == 200:384 temp_file.seek(0)385 temp_file.truncate()386 resume_size = 0387 388 total: int | None = _get_file_length_from_http_response(response)389 390 if displayed_filename is None:391 displayed_filename = url392 content_disposition = response.headers.get("Content-Disposition")393 if content_disposition is not None:394 match = HEADER_FILENAME_PATTERN.search(content_disposition)395 if match is not None:396 # Means file is on CDN397 displayed_filename = match.groupdict()["filename"]398 399 # Truncate filename if too long to display400 if len(displayed_filename) > 40:401 displayed_filename = f"(…){displayed_filename[-40:]}"402 403 consistency_error_message = (404 f"Consistency check failed: file should be of size {expected_size} but has size"405 f" {{actual_size}} ({displayed_filename}).\nThis is usually due to network issues while downloading the file."406 " Please retry with `force_download=True`."407 )408 progress_cm = _get_progress_bar_context(409 desc=displayed_filename,410 log_level=logger.getEffectiveLevel(),411 total=total,412 initial=resume_size,413 name="huggingface_hub.http_get",414 tqdm_class=tqdm_class,415 _tqdm_bar=_tqdm_bar,416 )417 418 with progress_cm as progress:419 new_resume_size = resume_size420 try:421 for chunk in response.iter_bytes(chunk_size=constants.DOWNLOAD_CHUNK_SIZE):422 if chunk: # filter out keep-alive new chunks423 progress.update(len(chunk))424 temp_file.write(chunk)425 new_resume_size += len(chunk)426 # Some data has been downloaded from the server so we reset the number of retries.427 _nb_retries = 5428 except (httpx.ConnectError, httpx.TimeoutException) as e:429 # If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely430 # a transient error (network outage?). We log a warning message and try to resume the download a few times431 # before giving up. Tre retry mechanism is basic but should be enough in most cases.432 if _nb_retries <= 0:433 logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e))434 raise435 logger.warning("Error while downloading from %s: %s\nTrying to resume download...", url, str(e))436 time.sleep(1)437 return http_get(438 url=url,439 temp_file=temp_file,440 resume_size=new_resume_size,441 headers=initial_headers,442 expected_size=expected_size,443 tqdm_class=tqdm_class,444 _nb_retries=_nb_retries - 1,445 _tqdm_bar=_tqdm_bar,446 )447 448 if expected_size is not None and expected_size != temp_file.tell():449 raise OSError(450 consistency_error_message.format(451 actual_size=temp_file.tell(),452 )453 )454 455 456def xet_get(457 *,458 incomplete_path: Path,459 xet_file_data: XetFileData,460 headers: dict[str, str],461 expected_size: int | None = None,462 displayed_filename: str | None = None,463 tqdm_class: type[base_tqdm] | None = None,464 _tqdm_bar: tqdm | None = None,465) -> None:466 """467 Download a file using Xet storage service.468 469 Args:470 incomplete_path (`Path`):471 The path to the file to download.472 xet_file_data (`XetFileData`):473 The file metadata needed to make the request to the xet storage service.474 headers (`dict[str, str]`):475 The headers to send to the xet storage service.476 expected_size (`int`, *optional*):477 The expected size of the file to download. If set, the download will raise an error if the size of the478 received content is different from the expected one.479 displayed_filename (`str`, *optional*):480 The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If481 not set, the filename is guessed from the URL or the `Content-Disposition` header.482 483 **How it works:**484 The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks485 for efficient storage and transfer.486 487 `hf_xet.download_files` manages downloading files by:488 - Taking a list of files to download (each with its unique content hash)489 - Connecting to a storage server (CAS server) that knows how files are chunked490 - Using authentication to ensure secure access491 - Providing progress updates during download492 493 Authentication works by regularly refreshing access tokens through `refresh_xet_connection_info` to maintain a valid494 connection to the storage server.495 496 The download process works like this:497 1. Create a local cache folder at `~/.cache/huggingface/xet/chunk-cache` to store reusable file chunks498 2. Download files in parallel:499 2.1. Prepare to write the file to disk500 2.2. Ask the server "how is this file split into chunks?" using the file's unique hash501 The server responds with:502 - Which chunks make up the complete file503 - Where each chunk can be downloaded from504 2.3. For each needed chunk:505 - Checks if we already have it in our local cache506 - If not, download it from cloud storage (S3)507 - Save it to cache for future use508 - Assemble the chunks in order to recreate the original file509 510 """511 try:512 from hf_xet import PyXetDownloadInfo, download_files # type: ignore[no-redef]513 except ImportError:514 raise ValueError(515 "To use optimized download using Xet storage, you need to install the hf_xet package. "516 'Try `pip install "huggingface_hub[hf_xet]"` or `pip install hf_xet`.'517 )518 519 connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers)520 521 def token_refresher() -> tuple[str, int]:522 connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers)523 if connection_info is None:524 raise ValueError("Failed to refresh token using xet metadata.")525 return connection_info.access_token, connection_info.expiration_unix_epoch526 527 xet_download_info = [528 PyXetDownloadInfo(529 destination_path=str(incomplete_path.absolute()), hash=xet_file_data.file_hash, file_size=expected_size530 )531 ]532 533 if not displayed_filename:534 displayed_filename = incomplete_path.name535 536 # Truncate filename if too long to display537 if len(displayed_filename) > 40:538 displayed_filename = f"{displayed_filename[:40]}(…)"539 540 progress_cm = _get_progress_bar_context(541 desc=displayed_filename,542 log_level=logger.getEffectiveLevel(),543 total=expected_size,544 initial=0,545 name="huggingface_hub.xet_get",546 tqdm_class=tqdm_class,547 _tqdm_bar=_tqdm_bar,548 )549 550 xet_headers = headers.copy()551 xet_headers.pop("authorization", None)552 553 with progress_cm as progress:554 555 def progress_updater(progress_bytes: float):556 progress.update(progress_bytes)557 558 download_files(559 xet_download_info,560 endpoint=connection_info.endpoint,561 token_info=(connection_info.access_token, connection_info.expiration_unix_epoch),562 token_refresher=token_refresher,563 progress_updater=[progress_updater],564 request_headers=xet_headers,565 )566 567 568def _normalize_etag(etag: str | None) -> str | None:569 """Normalize ETag HTTP header, so it can be used to create nice filepaths.570 571 The HTTP spec allows two forms of ETag:572 ETag: W/"<etag_value>"573 ETag: "<etag_value>"574 575 For now, we only expect the second form from the server, but we want to be future-proof so we support both. For576 more context, see `TestNormalizeEtag` tests and https://github.com/huggingface/huggingface_hub/pull/1428.577 578 Args:579 etag (`str`, *optional*): HTTP header580 581 Returns:582 `str` or `None`: string that can be used as a nice directory name.583 Returns `None` if input is None.584 """585 if etag is None:586 return None587 return etag.lstrip("W/").strip('"')588 589 590def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None:591 """Alias method used in `transformers` conversion script."""592 return _create_symlink(src=src, dst=dst, new_blob=new_blob)593 594 595def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:596 """Create a symbolic link named dst pointing to src.597 598 By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages:599 - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will600 not break.601 - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when602 changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398,603 https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228.604 NOTE: The issue with absolute paths doesn't happen on admin mode.605 When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created.606 This happens when paths are not on the same volume. In that case, we use absolute paths.607 608 609 The result layout looks something like610 └── [ 128] snapshots611 ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f612 │ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812613 │ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd614 615 If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by616 having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file617 (`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing618 cache, the file is duplicated on the disk.619 620 In case symlinks are not supported, a warning message is displayed to the user once when loading `huggingface_hub`.621 The warning message can be disabled with the `DISABLE_SYMLINKS_WARNING` environment variable.622 """623 try:624 os.remove(dst)625 except OSError:626 pass627 628 abs_src = os.path.abspath(os.path.expanduser(src))629 abs_dst = os.path.abspath(os.path.expanduser(dst))630 abs_dst_folder = os.path.dirname(abs_dst)631 632 # Use relative_dst in priority633 try:634 relative_src = os.path.relpath(abs_src, abs_dst_folder)635 except ValueError:636 # Raised on Windows if src and dst are not on the same volume. This is the case when creating a symlink to a637 # local_dir instead of within the cache directory.638 # See https://docs.python.org/3/library/os.path.html#os.path.relpath639 relative_src = None640 641 try:642 commonpath = os.path.commonpath([abs_src, abs_dst])643 _support_symlinks = are_symlinks_supported(commonpath)644 except ValueError:645 # Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos.646 # See https://docs.python.org/3/library/os.path.html#os.path.commonpath647 _support_symlinks = os.name != "nt" and not constants.HF_HUB_DISABLE_SYMLINKS648 except PermissionError:649 # Permission error means src and dst are not in the same volume (e.g. destination path has been provided650 # by the user via `local_dir`. Let's test symlink support there)651 _support_symlinks = are_symlinks_supported(abs_dst_folder)652 except OSError as e:653 # OS error (errno=30) means that the commonpath is readonly on Linux/MacOS.654 if e.errno == errno.EROFS:655 _support_symlinks = are_symlinks_supported(abs_dst_folder)656 else:657 raise658 659 # Symlinks are supported => let's create a symlink.660 if _support_symlinks:661 src_rel_or_abs = relative_src or abs_src662 logger.debug(f"Creating pointer from {src_rel_or_abs} to {abs_dst}")663 try:664 os.symlink(src_rel_or_abs, abs_dst)665 return666 except FileExistsError:667 if os.path.islink(abs_dst) and os.path.realpath(abs_dst) == os.path.realpath(abs_src):668 # `abs_dst` already exists and is a symlink to the `abs_src` blob. It is most likely that the file has669 # been cached twice concurrently (exactly between `os.remove` and `os.symlink`). Do nothing.670 return671 else:672 # Very unlikely to happen. Means a file `dst` has been created exactly between `os.remove` and673 # `os.symlink` and is not a symlink to the `abs_src` blob file. Raise exception.674 raise675 except PermissionError:676 # Permission error means src and dst are not in the same volume (e.g. download to local dir) and symlink677 # is supported on both volumes but not between them. Let's just make a hard copy in that case.678 pass679 680 # Symlinks are not supported => let's move or copy the file.681 if new_blob:682 logger.debug(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}")683 shutil.move(abs_src, abs_dst, copy_function=_copy_no_matter_what)684 else:685 logger.debug(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}")686 shutil.copyfile(abs_src, abs_dst)687 688 689def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None:690 """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash.691 692 Does nothing if `revision` is already a proper `commit_hash` or reference is already cached.693 """694 if revision != commit_hash:695 ref_path = Path(storage_folder) / "refs" / revision696 ref_path.parent.mkdir(parents=True, exist_ok=True)697 if not ref_path.exists() or commit_hash != ref_path.read_text():698 # Update ref only if has been updated. Could cause useless error in case699 # repo is already cached and user doesn't have write access to cache folder.700 # See https://github.com/huggingface/huggingface_hub/issues/1216.701 ref_path.write_text(commit_hash)702 703 704@validate_hf_hub_args705def repo_folder_name(*, repo_id: str, repo_type: str) -> str:706 """Return a serialized version of a hf.co repo name and type, safe for disk storage707 as a single non-nested folder.708 709 Example: models--julien-c--EsperBERTo-small710 """711 # remove all `/` occurrences to correctly convert repo to directory name712 parts = [f"{repo_type}s", *repo_id.split("/")]713 return constants.REPO_ID_SEPARATOR.join(parts)714 715 716def _check_disk_space(expected_size: int, target_dir: str | Path) -> None:717 """Check disk usage and log a warning if there is not enough disk space to download the file.718 719 Args:720 expected_size (`int`):721 The expected size of the file in bytes.722 target_dir (`str`):723 The directory where the file will be stored after downloading.724 """725 726 target_dir = Path(target_dir) # format as `Path`727 for path in [target_dir] + list(target_dir.parents): # first check target_dir, then each parents one by one728 try:729 target_dir_free = shutil.disk_usage(path).free730 if target_dir_free < expected_size:731 warnings.warn(732 "Not enough free disk space to download the file. "733 f"The expected file size is: {expected_size / 1e6:.2f} MB. "734 f"The target location {target_dir} only has {target_dir_free / 1e6:.2f} MB free disk space."735 )736 return737 except OSError: # raise on anything: file does not exist or space disk cannot be checked738 pass739 740 741@overload742def hf_hub_download(743 repo_id: str,744 filename: str,745 *,746 subfolder: str | None = None,747 repo_type: str | None = None,748 revision: str | None = None,749 library_name: str | None = None,750 library_version: str | None = None,751 cache_dir: str | Path | None = None,752 local_dir: str | Path | None = None,753 user_agent: dict | str | None = None,754 force_download: bool = False,755 etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,756 token: bool | str | None = None,757 local_files_only: bool = False,758 headers: dict[str, str] | None = None,759 endpoint: str | None = None,760 tqdm_class: type[base_tqdm] | None = None,761 dry_run: Literal[False] = False,762) -> str: ...763 764 765@overload766def hf_hub_download(767 repo_id: str,768 filename: str,769 *,770 subfolder: str | None = None,771 repo_type: str | None = None,772 revision: str | None = None,773 library_name: str | None = None,774 library_version: str | None = None,775 cache_dir: str | Path | None = None,776 local_dir: str | Path | None = None,777 user_agent: dict | str | None = None,778 force_download: bool = False,779 etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,780 token: bool | str | None = None,781 local_files_only: bool = False,782 headers: dict[str, str] | None = None,783 endpoint: str | None = None,784 tqdm_class: type[base_tqdm] | None = None,785 dry_run: Literal[True] = True,786) -> DryRunFileInfo: ...787 788 789@overload790def hf_hub_download(791 repo_id: str,792 filename: str,793 *,794 subfolder: str | None = None,795 repo_type: str | None = None,796 revision: str | None = None,797 library_name: str | None = None,798 library_version: str | None = None,799 cache_dir: str | Path | None = None,800 local_dir: str | Path | None = None,801 user_agent: dict | str | None = None,802 force_download: bool = False,803 etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,804 token: bool | str | None = None,805 local_files_only: bool = False,806 headers: dict[str, str] | None = None,807 endpoint: str | None = None,808 tqdm_class: type[base_tqdm] | None = None,809 dry_run: bool = False,810) -> str | DryRunFileInfo: ...811 812 813@validate_hf_hub_args814def hf_hub_download(815 repo_id: str,816 filename: str,817 *,818 subfolder: str | None = None,819 repo_type: str | None = None,820 revision: str | None = None,821 library_name: str | None = None,822 library_version: str | None = None,823 cache_dir: str | Path | None = None,824 local_dir: str | Path | None = None,825 user_agent: dict | str | None = None,826 force_download: bool = False,827 etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,828 token: bool | str | None = None,829 local_files_only: bool = False,830 headers: dict[str, str] | None = None,831 endpoint: str | None = None,832 tqdm_class: type[base_tqdm] | None = None,833 dry_run: bool = False,834) -> str | DryRunFileInfo:835 """Download a given file if it's not already present in the local cache.836 837 The new cache file layout looks like this:838 - The cache directory contains one subfolder per repo_id (namespaced by repo type)839 - inside each repo folder:840 - refs is a list of the latest known revision => commit_hash pairs841 - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on842 whether they're LFS files or not)843 - snapshots contains one subfolder per commit, each "commit" contains the subset of the files844 that have been resolved at that particular commit. Each filename is a symlink to the blob845 at that particular commit.846 847 ```848 [ 96] .849 └── [ 160] models--julien-c--EsperBERTo-small850 ├── [ 160] blobs851 │ ├── [321M] 403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd852 │ ├── [ 398] 7cb18dc9bafbfcf74629a4b760af1b160957a83e853 │ └── [1.4K] d7edf6bd2a681fb0175f7735299831ee1b22b812854 ├── [ 96] refs855 │ └── [ 40] main856 └── [ 128] snapshots857 ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f858 │ ├── [ 52] README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812859 │ └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd860 └── [ 128] bbc77c8132af1cc5cf678da3f1ddf2de43606d48861 ├── [ 52] README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e862 └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd863 ```864 865 If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this866 option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir`867 to store some metadata related to the downloaded files. While this mechanism is not as robust as the main868 cache-system, it's optimized for regularly pulling the latest version of a repository.869 870 Args:871 repo_id (`str`):872 A user or an organization name and a repo name separated by a `/`.873 filename (`str`):874 The name of the file in the repo.875 subfolder (`str`, *optional*):876 An optional value corresponding to a folder inside the model repo.877 repo_type (`str`, *optional*):878 Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo,879 `None` or `"model"` if downloading from a model. Default is `None`.880 revision (`str`, *optional*):881 An optional Git revision id which can be a branch name, a tag, or a882 commit hash.883 library_name (`str`, *optional*):884 The name of the library to which the object corresponds.885 library_version (`str`, *optional*):886 The version of the library.887 cache_dir (`str`, `Path`, *optional*):888 Path to the folder where cached files are stored.889 local_dir (`str` or `Path`, *optional*):890 If provided, the downloaded file will be placed under this directory.891 user_agent (`dict`, `str`, *optional*):892 The user-agent info in the form of a dictionary or a string.893 force_download (`bool`, *optional*, defaults to `False`):894 Whether the file should be downloaded even if it already exists in895 the local cache.896 etag_timeout (`float`, *optional*, defaults to `10`):897 When fetching ETag, how many seconds to wait for the server to send898 data before giving up which is passed to `requests.request`.899 token (`str`, `bool`, *optional*):900 A token to be used for the download.901 - If `True`, the token is read from the HuggingFace config902 folder.903 - If a string, it's used as the authentication token.904 local_files_only (`bool`, *optional*, defaults to `False`):905 If `True`, avoid downloading the file and return the path to the906 local cached file if it exists.907 headers (`dict`, *optional*):908 Additional headers to be sent with the request.909 tqdm_class (`tqdm`, *optional*):910 If provided, overwrites the default behavior for the progress bar. Passed911 argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior.912 Defaults to the custom HF progress bar that can be disabled by setting913 `HF_HUB_DISABLE_PROGRESS_BARS` environment variable.914 dry_run (`bool`, *optional*, defaults to `False`):915 If `True`, perform a dry run without actually downloading the file. Returns a916 [`DryRunFileInfo`] object containing information about what would be downloaded.917 918 Returns:919 `str` or [`DryRunFileInfo`]:920 - If `dry_run=False`: Local path of file or if networking is off, last version of file cached on disk.921 - If `dry_run=True`: A [`DryRunFileInfo`] object containing download information.922 923 Raises:924 [`~utils.RepositoryNotFoundError`]925 If the repository to download from cannot be found. This may be because it doesn't exist,926 or because it is set to `private` and you do not have access.927 [`~utils.RevisionNotFoundError`]928 If the revision to download from cannot be found.929 [`~utils.RemoteEntryNotFoundError`]930 If the file to download cannot be found.931 [`~utils.LocalEntryNotFoundError`]932 If network is disabled or unavailable and file is not found in cache.933 [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)934 If `token=True` but the token cannot be found.935 [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)936 If ETag cannot be determined.937 [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)938 If some parameter value is invalid.939 940 """941 if constants.HF_HUB_ETAG_TIMEOUT != constants.DEFAULT_ETAG_TIMEOUT:942 # Respect environment variable above user value943 etag_timeout = constants.HF_HUB_ETAG_TIMEOUT944 945 if cache_dir is None:946 cache_dir = constants.HF_HUB_CACHE947 if revision is None:948 revision = constants.DEFAULT_REVISION949 if isinstance(cache_dir, Path):950 cache_dir = str(cache_dir)951 if isinstance(local_dir, Path):952 local_dir = str(local_dir)953 954 if subfolder == "":955 subfolder = None956 if subfolder is not None:957 # This is used to create a URL, and not a local path, hence the forward slash.958 filename = f"{subfolder}/{filename}"959 960 if repo_type is None:961 repo_type = "model"962 if repo_type not in constants.REPO_TYPES_WITH_KERNEL:963 raise ValueError(964 f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}"965 )966 967 hf_headers = build_hf_headers(968 token=token,969 library_name=library_name,970 library_version=library_version,971 user_agent=user_agent,972 headers=headers,973 )974 975 if local_dir is not None:976 return _hf_hub_download_to_local_dir(977 # Destination978 local_dir=local_dir,979 # File info980 repo_id=repo_id,981 repo_type=repo_type,982 filename=filename,983 revision=revision,984 # HTTP info985 endpoint=endpoint,986 etag_timeout=etag_timeout,987 headers=hf_headers,988 token=token,989 # Additional options990 cache_dir=cache_dir,991 force_download=force_download,992 local_files_only=local_files_only,993 tqdm_class=tqdm_class,994 dry_run=dry_run,995 )996 else:997 return _hf_hub_download_to_cache_dir(998 # Destination999 cache_dir=cache_dir,1000 # File info1001 repo_id=repo_id,1002 filename=filename,1003 repo_type=repo_type,1004 revision=revision,1005 # HTTP info1006 endpoint=endpoint,1007 etag_timeout=etag_timeout,1008 headers=hf_headers,1009 token=token,1010 # Additional options1011 local_files_only=local_files_only,1012 force_download=force_download,1013 tqdm_class=tqdm_class,1014 dry_run=dry_run,1015 )1016 1017 1018def _hf_hub_download_to_cache_dir(1019 *,1020 # Destination1021 cache_dir: str,1022 # File info1023 repo_id: str,1024 filename: str,1025 repo_type: str,1026 revision: str,1027 # HTTP info1028 endpoint: str | None,1029 etag_timeout: float,1030 headers: dict[str, str],1031 token: bool | str | None,1032 # Additional options1033 local_files_only: bool,1034 force_download: bool,1035 tqdm_class: type[base_tqdm] | None,1036 dry_run: bool,1037) -> str | DryRunFileInfo:1038 """Download a given file to a cache folder, if not already present.1039 1040 Method should not be called directly. Please use `hf_hub_download` instead.1041 """1042 locks_dir = os.path.join(cache_dir, ".locks")1043 storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))1044 1045 # cross-platform transcription of filename, to be used as a local file path.1046 relative_filename = os.path.join(*filename.split("/"))1047 if os.name == "nt":1048 if relative_filename.startswith("..\\") or "\\..\\" in relative_filename:1049 raise ValueError(1050 f"Invalid filename: cannot handle filename '{relative_filename}' on Windows. Please ask the repository"1051 " owner to rename this file."1052 )1053 1054 # if user provides a commit_hash and they already have the file on disk, shortcut everything.1055 if REGEX_COMMIT_HASH.match(revision):1056 pointer_path = _get_pointer_path(storage_folder, revision, relative_filename)1057 if os.path.exists(pointer_path):1058 if dry_run:1059 return DryRunFileInfo(1060 commit_hash=revision,1061 file_size=os.path.getsize(pointer_path),1062 filename=filename,1063 is_cached=True,1064 local_path=pointer_path,1065 will_download=force_download,1066 )1067 if not force_download:1068 return pointer_path1069 1070 # Try to get metadata (etag, commit_hash, url, size) from the server.1071 # If we can't, a HEAD request error is returned.1072 (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error(1073 repo_id=repo_id,1074 filename=filename,1075 repo_type=repo_type,1076 revision=revision,1077 endpoint=endpoint,1078 etag_timeout=etag_timeout,1079 headers=headers,1080 token=token,1081 local_files_only=local_files_only,1082 storage_folder=storage_folder,1083 relative_filename=relative_filename,1084 )1085 1086 # etag can be None for several reasons:1087 # 1. we passed local_files_only.1088 # 2. we don't have a connection1089 # 3. Hub is down (HTTP 500, 503, 504)1090 # 4. repo is not found -for example private or gated- and invalid/missing token sent1091 # 5. Hub is blocked by a firewall or proxy is not set correctly.1092 # => Try to get the last downloaded one from the specified revision.1093 #1094 # If the specified revision is a commit hash, look inside "snapshots".1095 # If the specified revision is a branch or tag, look inside "refs".1096 if head_call_error is not None:1097 # Couldn't make a HEAD call => let's try to find a local file1098 if not force_download:1099 commit_hash = None1100 if REGEX_COMMIT_HASH.match(revision):1101 commit_hash = revision1102 else:1103 ref_path = os.path.join(storage_folder, "refs", revision)1104 if os.path.isfile(ref_path):1105 with open(ref_path) as f:1106 commit_hash = f.read()1107 1108 # Return pointer file if exists1109 if commit_hash is not None:1110 pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)1111 if os.path.exists(pointer_path):1112 if dry_run:1113 return DryRunFileInfo(1114 commit_hash=commit_hash,1115 file_size=os.path.getsize(pointer_path),1116 filename=filename,1117 is_cached=True,1118 local_path=pointer_path,1119 will_download=force_download,1120 )1121 if not force_download:1122 return pointer_path1123 1124 if isinstance(head_call_error, _DEFAULT_RETRY_ON_EXCEPTIONS) or (1125 isinstance(head_call_error, HfHubHTTPError)1126 and head_call_error.response.status_code in _DEFAULT_RETRY_ON_STATUS_CODES1127 ):1128 logger.info("No local file found. Retrying..")1129 (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = (1130 _get_metadata_or_catch_error(1131 repo_id=repo_id,1132 filename=filename,1133 repo_type=repo_type,1134 revision=revision,1135 endpoint=endpoint,1136 etag_timeout=_ETAG_RETRY_TIMEOUT,1137 headers=headers,1138 token=token,1139 local_files_only=local_files_only,1140 storage_folder=storage_folder,1141 relative_filename=relative_filename,1142 retry_on_errors=True,1143 )1144 )1145 1146 # If still error, raise1147 if head_call_error is not None:1148 _raise_on_head_call_error(head_call_error, force_download, local_files_only)1149 1150 # From now on, etag, commit_hash, url and size are not None.1151 assert etag is not None, "etag must have been retrieved from server"1152 assert commit_hash is not None, "commit_hash must have been retrieved from server"1153 assert url_to_download is not None, "file location must have been retrieved from server"1154 assert expected_size is not None, "expected_size must have been retrieved from server"1155 blob_path = os.path.join(storage_folder, "blobs", etag)1156 pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename)1157 1158 if dry_run:1159 is_cached = os.path.exists(pointer_path) or os.path.exists(blob_path)1160 return DryRunFileInfo(1161 commit_hash=commit_hash,1162 file_size=expected_size,1163 filename=filename,1164 is_cached=is_cached,1165 local_path=pointer_path,1166 will_download=force_download or not is_cached,1167 )1168 1169 os.makedirs(os.path.dirname(blob_path), exist_ok=True)1170 os.makedirs(os.path.dirname(pointer_path), exist_ok=True)1171 1172 # Tag cache_dir so backup tools can skip it (CACHEDIR.TAG standard).1173 _create_cachedir_tag(Path(cache_dir))1174 1175 # if passed revision is not identical to commit_hash1176 # then revision has to be a branch name or tag name.1177 # In that case store a ref.1178 _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)1179 1180 # Prevent parallel downloads of the same file with a lock.1181 # etag could be duplicated across repos,1182 lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock")1183 1184 # Some Windows versions do not allow for paths longer than 255 characters.1185 # In this case, we must specify it as an extended path by using the "\\?\" prefix.1186 if (1187 os.name == "nt"1188 and len(os.path.abspath(lock_path)) > 2551189 and not os.path.abspath(lock_path).startswith("\\\\?\\")1190 ):1191 lock_path = "\\\\?\\" + os.path.abspath(lock_path)1192 1193 if (1194 os.name == "nt"1195 and len(os.path.abspath(blob_path)) > 2551196 and not os.path.abspath(blob_path).startswith("\\\\?\\")1197 ):1198 blob_path = "\\\\?\\" + os.path.abspath(blob_path)1199 1200 Path(lock_path).parent.mkdir(parents=True, exist_ok=True)