codekingpro/portable-devtools

1# Copyright 2024-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Contains utilities to handle the `../.cache/huggingface` folder in local directories.15 16First discussed in https://github.com/huggingface/huggingface_hub/issues/1738 to store17download metadata when downloading files from the hub to a local directory (without18using the cache).19 20./.cache/huggingface folder structure:21[4.0K]  data22├── [4.0K]  .cache23│   └── [4.0K]  huggingface24│       └── [4.0K]  download25│           ├── [  16]  file.parquet.metadata26│           ├── [  16]  file.txt.metadata27│           └── [4.0K]  folder28│               └── [  16]  file.parquet.metadata29│30├── [6.5G]  file.parquet31├── [1.5K]  file.txt32└── [4.0K]  folder33    └── [   16]  file.parquet34 35 36Download metadata file structure:37```38# file.txt.metadata3911c5a3d5811f50298f278a704980280950aedb1040a16a55fda99d2f2e7b69cce5cf93ff4ad3049930411712656091.12342 43# file.parquet.metadata4411c5a3d5811f50298f278a704980280950aedb10457c5d3f4b8b76583b422fcb9189ad6c89d5d97a094541ce8932dce3ecabde1421461712656091.12347}48```49"""50 51import base6452import hashlib53import logging54import os55import time56from dataclasses import dataclass57from pathlib import Path58 59from .utils import WeakFileLock60 61 62logger = logging.getLogger(__name__)63 64CACHEDIR_TAG_CONTENT = (65    "Signature: 8a477f597d28d172789f06886806bc55\n"66    "# This file is a cache directory tag created by huggingface_hub.\n"67    "# For information about cache directory tags, see:\n"68    "#\thttps://bford.info/cachedir/\n"69)70 71 72@dataclass73class LocalDownloadFilePaths:74    """75    Paths to the files related to a download process in a local dir.76 77    Returned by [`get_local_download_paths`].78 79    Attributes:80        file_path (`Path`):81            Path where the file will be saved.82        lock_path (`Path`):83            Path to the lock file used to ensure atomicity when reading/writing metadata.84        metadata_path (`Path`):85            Path to the metadata file.86    """87 88    file_path: Path89    lock_path: Path90    metadata_path: Path91 92    def incomplete_path(self, etag: str) -> Path:93        """Return the path where a file will be temporarily downloaded before being moved to `file_path`."""94        path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete"95        resolved_path = str(path.resolve())96        # Some Windows versions do not allow for paths longer than 255 characters.97        # In this case, we must specify it as an extended path by using the "\\?\" prefix.98        if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"):99            path = Path("\\\\?\\" + resolved_path)100        return path101 102 103@dataclass(frozen=True)104class LocalUploadFilePaths:105    """106    Paths to the files related to an upload process in a local dir.107 108    Returned by [`get_local_upload_paths`].109 110    Attributes:111        path_in_repo (`str`):112            Path of the file in the repo.113        file_path (`Path`):114            Path where the file will be saved.115        lock_path (`Path`):116            Path to the lock file used to ensure atomicity when reading/writing metadata.117        metadata_path (`Path`):118            Path to the metadata file.119    """120 121    path_in_repo: str122    file_path: Path123    lock_path: Path124    metadata_path: Path125 126 127@dataclass128class LocalDownloadFileMetadata:129    """130    Metadata about a file in the local directory related to a download process.131 132    Attributes:133        filename (`str`):134            Path of the file in the repo.135        commit_hash (`str`):136            Commit hash of the file in the repo.137        etag (`str`):138            ETag of the file in the repo. Used to check if the file has changed.139            For LFS files, this is the sha256 of the file. For regular files, it corresponds to the git hash.140        timestamp (`int`):141            Unix timestamp of when the metadata was saved i.e. when the metadata was accurate.142    """143 144    filename: str145    commit_hash: str146    etag: str147    timestamp: float148 149 150@dataclass151class LocalUploadFileMetadata:152    """153    Metadata about a file in the local directory related to an upload process.154    """155 156    size: int157 158    # Default values correspond to "we don't know yet"159    timestamp: float | None = None160    should_ignore: bool | None = None161    sha256: str | None = None162    upload_mode: str | None = None163    remote_oid: str | None = None164    is_uploaded: bool = False165    is_committed: bool = False166 167    def save(self, paths: LocalUploadFilePaths) -> None:168        """Save the metadata to disk."""169        with WeakFileLock(paths.lock_path):170            with paths.metadata_path.open("w") as f:171                new_timestamp = time.time()172                f.write(str(new_timestamp) + "\n")173 174                f.write(str(self.size))  # never None175                f.write("\n")176 177                if self.should_ignore is not None:178                    f.write(str(int(self.should_ignore)))179                f.write("\n")180 181                if self.sha256 is not None:182                    f.write(self.sha256)183                f.write("\n")184 185                if self.upload_mode is not None:186                    f.write(self.upload_mode)187                f.write("\n")188 189                if self.remote_oid is not None:190                    f.write(self.remote_oid)191                f.write("\n")192 193                f.write(str(int(self.is_uploaded)) + "\n")194                f.write(str(int(self.is_committed)) + "\n")195 196            self.timestamp = new_timestamp197 198 199def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFilePaths:200    """Compute paths to the files related to a download process.201 202    Folders containing the paths are all guaranteed to exist.203 204    Args:205        local_dir (`Path`):206            Path to the local directory in which files are downloaded.207        filename (`str`):208            Path of the file in the repo.209 210    Return:211        [`LocalDownloadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path, incomplete_path).212    """213    # filename is the path in the Hub repository (separated by '/')214    # make sure to have a cross-platform transcription215    sanitized_filename = os.path.join(*filename.split("/"))216    if os.name == "nt":217        if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename:218            raise ValueError(219                f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository"220                " owner to rename this file."221            )222    file_path = local_dir / sanitized_filename223    metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata"224    lock_path = metadata_path.with_suffix(".lock")225 226    # Some Windows versions do not allow for paths longer than 255 characters.227    # In this case, we must specify it as an extended path by using the "\\?\" prefix228    if os.name == "nt":229        if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255:230            file_path = Path("\\\\?\\" + os.path.abspath(file_path))231            lock_path = Path("\\\\?\\" + os.path.abspath(lock_path))232            metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path))233 234    file_path.parent.mkdir(parents=True, exist_ok=True)235    metadata_path.parent.mkdir(parents=True, exist_ok=True)236    return LocalDownloadFilePaths(file_path=file_path, lock_path=lock_path, metadata_path=metadata_path)237 238 239def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePaths:240    """Compute paths to the files related to an upload process.241 242    Folders containing the paths are all guaranteed to exist.243 244    Args:245        local_dir (`Path`):246            Path to the local directory that is uploaded.247        filename (`str`):248            Path of the file in the repo.249 250    Return:251        [`LocalUploadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path).252    """253    # filename is the path in the Hub repository (separated by '/')254    # make sure to have a cross-platform transcription255    sanitized_filename = os.path.join(*filename.split("/"))256    if os.name == "nt":257        if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename:258            raise ValueError(259                f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository"260                " owner to rename this file."261            )262    file_path = local_dir / sanitized_filename263    metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata"264    lock_path = metadata_path.with_suffix(".lock")265 266    # Some Windows versions do not allow for paths longer than 255 characters.267    # In this case, we must specify it as an extended path by using the "\\?\" prefix268    if os.name == "nt":269        if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255:270            file_path = Path("\\\\?\\" + os.path.abspath(file_path))271            lock_path = Path("\\\\?\\" + os.path.abspath(lock_path))272            metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path))273 274    file_path.parent.mkdir(parents=True, exist_ok=True)275    metadata_path.parent.mkdir(parents=True, exist_ok=True)276    return LocalUploadFilePaths(277        path_in_repo=filename, file_path=file_path, lock_path=lock_path, metadata_path=metadata_path278    )279 280 281def read_download_metadata(local_dir: Path, filename: str) -> LocalDownloadFileMetadata | None:282    """Read metadata about a file in the local directory related to a download process.283 284    Args:285        local_dir (`Path`):286            Path to the local directory in which files are downloaded.287        filename (`str`):288            Path of the file in the repo.289 290    Return:291        `[LocalDownloadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise.292    """293    paths = get_local_download_paths(local_dir, filename)294    with WeakFileLock(paths.lock_path):295        if paths.metadata_path.exists():296            try:297                with paths.metadata_path.open() as f:298                    commit_hash = f.readline().strip()299                    etag = f.readline().strip()300                    timestamp = float(f.readline().strip())301                    metadata = LocalDownloadFileMetadata(302                        filename=filename,303                        commit_hash=commit_hash,304                        etag=etag,305                        timestamp=timestamp,306                    )307            except Exception as e:308                # remove the metadata file if it is corrupted / not the right format309                logger.warning(310                    f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue."311                )312                try:313                    paths.metadata_path.unlink()314                except Exception as e:315                    logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}")316                return None317 318            try:319                # check if the file exists and hasn't been modified since the metadata was saved320                stat = paths.file_path.stat()321                if (322                    stat.st_mtime - 1 <= metadata.timestamp323                ):  # allow 1s difference as stat.st_mtime might not be precise324                    return metadata325                logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.")326            except FileNotFoundError:327                # file does not exist => metadata is outdated328                return None329    return None330 331 332def read_upload_metadata(local_dir: Path, filename: str) -> LocalUploadFileMetadata:333    """Read metadata about a file in the local directory related to an upload process.334 335    TODO: factorize logic with `read_download_metadata`.336 337    Args:338        local_dir (`Path`):339            Path to the local directory in which files are downloaded.340        filename (`str`):341            Path of the file in the repo.342 343    Return:344        `[LocalUploadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise.345    """346    paths = get_local_upload_paths(local_dir, filename)347    with WeakFileLock(paths.lock_path):348        if paths.metadata_path.exists():349            try:350                with paths.metadata_path.open() as f:351                    timestamp = float(f.readline().strip())352 353                    size = int(f.readline().strip())  # never None354 355                    _should_ignore = f.readline().strip()356                    should_ignore = None if _should_ignore == "" else bool(int(_should_ignore))357 358                    _sha256 = f.readline().strip()359                    sha256 = None if _sha256 == "" else _sha256360 361                    _upload_mode = f.readline().strip()362                    upload_mode = None if _upload_mode == "" else _upload_mode363                    if upload_mode not in (None, "regular", "lfs"):364                        raise ValueError(f"Invalid upload mode in metadata {paths.path_in_repo}: {upload_mode}")365 366                    _remote_oid = f.readline().strip()367                    remote_oid = None if _remote_oid == "" else _remote_oid368 369                    is_uploaded = bool(int(f.readline().strip()))370                    is_committed = bool(int(f.readline().strip()))371 372                    metadata = LocalUploadFileMetadata(373                        timestamp=timestamp,374                        size=size,375                        should_ignore=should_ignore,376                        sha256=sha256,377                        upload_mode=upload_mode,378                        remote_oid=remote_oid,379                        is_uploaded=is_uploaded,380                        is_committed=is_committed,381                    )382            except Exception as e:383                # remove the metadata file if it is corrupted / not the right format384                logger.warning(385                    f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue."386                )387                try:388                    paths.metadata_path.unlink()389                except Exception as e:390                    logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}")391 392                # corrupted metadata => we don't know anything expect its size393                return LocalUploadFileMetadata(size=paths.file_path.stat().st_size)394 395            # TODO: can we do better?396            if (397                metadata.timestamp is not None398                and metadata.is_uploaded  # file was uploaded399                and not metadata.is_committed  # but not committed400                and time.time() - metadata.timestamp > 20 * 3600  # and it's been more than 20 hours401            ):  # => we consider it as garbage-collected by S3402                metadata.is_uploaded = False403 404            # check if the file exists and hasn't been modified since the metadata was saved405            try:406                if metadata.timestamp is not None and paths.file_path.stat().st_mtime <= metadata.timestamp:407                    return metadata408                logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.")409            except FileNotFoundError:410                # file does not exist => metadata is outdated411                pass412 413    # empty metadata => we don't know anything expect its size414    return LocalUploadFileMetadata(size=paths.file_path.stat().st_size)415 416 417def write_download_metadata(local_dir: Path, filename: str, commit_hash: str, etag: str) -> None:418    """Write metadata about a file in the local directory related to a download process.419 420    Args:421        local_dir (`Path`):422            Path to the local directory in which files are downloaded.423    """424    paths = get_local_download_paths(local_dir, filename)425    with WeakFileLock(paths.lock_path):426        with paths.metadata_path.open("w") as f:427            f.write(f"{commit_hash}\n{etag}\n{time.time()}\n")428 429 430def _huggingface_dir(local_dir: Path) -> Path:431    """Return the path to the `.cache/huggingface` directory in a local directory."""432    # Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times433    path = local_dir / ".cache" / "huggingface"434    path.mkdir(exist_ok=True, parents=True)435 436    # Create a CACHEDIR.TAG so backup tools can skip this directory.437    _create_cachedir_tag(path)438 439    # Create a .gitignore file in the .cache/huggingface directory if it doesn't exist440    # Should be thread-safe enough like this.441    gitignore = path / ".gitignore"442    gitignore_lock = path / ".gitignore.lock"443    if not gitignore.exists():444        try:445            with WeakFileLock(gitignore_lock, timeout=0.1):446                gitignore.write_text("*")447        except IndexError:448            pass449        except OSError:  # TimeoutError, FileNotFoundError, PermissionError, etc.450            pass451        try:452            gitignore_lock.unlink()453        except OSError:454            pass455    return path456 457 458def _create_cachedir_tag(cache_dir: Path) -> None:459    """Create a CACHEDIR.TAG file in ``cache_dir`` if one does not already exist.460 461    The tag follows the `Cache Directory Tagging Standard <http://www.brynosaurus.com/cachedir/>`_462    so that backup tools can recognize and skip cache directories.463    """464    tag_path = cache_dir / "CACHEDIR.TAG"465    if not tag_path.exists():466        try:467            tag_path.write_text(CACHEDIR_TAG_CONTENT)468        except OSError:469            pass470 471 472def _short_hash(filename: str) -> str:473    return base64.urlsafe_b64encode(hashlib.sha1(filename.encode()).digest()).decode()474