codekingpro/portable-devtools

1# Copyright 2026-present, the HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Centralized parser for Hugging Face Hub URIs ('hf://...') and mount specifications.15 16A HF URI is a URI-like string that identifies a location on the Hugging Face17Hub: a model/dataset/space/kernel repository, a bucket, optionally a revision,18and optionally a path inside the repo or bucket.19 20Canonical syntax:21 22```23hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]24```25 26A HF mount wraps a HF URI with a local mount path and an optional ':ro'/':rw'27flag (used by Spaces and Jobs volumes):28 29```30hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]31```32 33See 'docs/source/en/package_reference/hf_uris.md' for the full grammar and examples.34"""35 36import re37from dataclasses import dataclass, field38from urllib.parse import unquote39 40from huggingface_hub import constants41from huggingface_hub.errors import HfUriError, HFValidationError42 43from ._validators import validate_repo_id44 45 46# Inverse map (singular -> plural URI prefix). Built once from the canonical47# 'constants.HF_URI_TYPE_PREFIXES' and used to render URIs.48_TYPE_TO_PREFIX: dict[str, str] = {v: k for k, v in constants.HF_URI_TYPE_PREFIXES.items()}49 50# Special revisions that contain a '/'. They take precedence when splitting51# the part after '@' into '<revision>/<path-in-repo>'. Matches 'refs/pr/N'52# (Pull Request refs) and 'refs/convert/<name>' (e.g. parquet conversions).53# The conversion name allows the typical git ref characters '[a-zA-Z0-9_.-]'54# so names like 'parquet-v2' or 'duckdb.v1' round-trip correctly.55_SPECIAL_REFS_REVISION_REGEX = re.compile(r"^refs/(?:convert/[\w.-]+|pr/\d+)")56 57# Same as constants.HfUriType, but as a set of strings for easy lookup.)58_VALID_URI_TYPES: frozenset[str] = frozenset(constants.HF_URI_TYPE_PREFIXES.values())59 60 61@dataclass(frozen=True)62class HfUri:63    """Parsed representation of a Hugging Face Hub URI ('hf://...').64 65    Attributes:66        type (`str`):67            One of 'model', 'dataset', 'space', 'kernel' or 'bucket'.68        id (`str`):69            The repository id ('namespace/name', e.g. 'my-org/my-model') for repo URIs, or the bucket id ('namespace/name') for bucket URIs.70        revision (`str`, *optional*):71            The revision specified after '@' in the URI, URL-decoded. 'None' if no revision was specified, or for bucket URIs (which72            never carry a revision). Special refs like 'refs/pr/10' and 'refs/convert/parquet' are preserved as-is.73        path_in_repo (`str`):74            The path inside the repo or bucket. Empty string if the URI points at the root.75    """76 77    type: constants.HfUriType78    id: str79    revision: str | None = None80    path_in_repo: str = ""81    _raw: str | None = field(repr=False, hash=False, compare=False, default=None)82 83    def __post_init__(self) -> None:84        uri = self._raw or ""  # For error messages85 86        # Check valid URI type87        if self.type not in _VALID_URI_TYPES:88            raise HfUriError(uri=uri, msg=f"Invalid type '{self.type}'. Must be one of {sorted(_VALID_URI_TYPES)}.")89 90        # Check valid ID91        if not self.id or self.id.count("/") != 1:92            raise HfUriError(uri=uri, msg=f"Id must be 'namespace/name', got '{self.id}'.")93        if self.type != "bucket":94            try:95                validate_repo_id(self.id)96            except HFValidationError as e:97                raise HfUriError(uri=uri, msg=str(e)) from e98 99        # Check valid revision100        if self.revision is not None and not self.revision:101            raise HfUriError(uri=uri, msg="Revision must not be an empty string.")102        if self.type == "bucket" and self.revision is not None:103            raise HfUriError(uri=uri, msg="Bucket URIs do not support a revision.")104 105        # Check valid path in repo106        if self.path_in_repo:107            if self.path_in_repo.startswith("/") or "//" in self.path_in_repo:108                raise HfUriError(uri=uri, msg=f"Path must not contain empty segments (got '{self.path_in_repo}').")109 110    @property111    def is_bucket(self) -> bool:112        """True if this URI points at a bucket."""113        return self.type == "bucket"114 115    @property116    def is_repo(self) -> bool:117        """True if this URI points at a repository (model, dataset, space or kernel)."""118        return self.type != "bucket"119 120    def to_uri(self) -> str:121        """Render the URI as a canonical 'hf://' string.122 123        The type prefix is always written explicitly (e.g. 'hf://models/my-org/my-model').124        """125        parts: list[str] = [constants.HF_PROTOCOL, _TYPE_TO_PREFIX[self.type], "/", self.id]126        if self.revision is not None:127            # Encode '/' as '%2F' for revisions that would otherwise be split as '<revision>/<path>'128            # at parse time. Special refs ('refs/pr/N', 'refs/convert/<name>') are kept verbatim129            # because the parser matches them eagerly.130            revision = self.revision131            if "/" in revision and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None:132                revision = revision.replace("/", "%2F")133            parts.append(f"@{revision}")134        if self.path_in_repo:135            parts.append(f"/{self.path_in_repo}")136        return "".join(parts)137 138 139@dataclass(frozen=True)140class HfMount:141    """A HF URI paired with a local mount path and optional read-only flag.142 143    Used by Spaces and Jobs to describe volume mounts. The full syntax is:144 145    ```146    hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]147    ```148 149    Attributes:150        source ([`HfUri`]):151            The parsed HF URI identifying the Hub resource to mount.152        mount_path (`str`):153            The local mount path (always starts with '/').154        read_only (`bool`, *optional*):155            True if the mount ends with ':ro', False if it ends with ':rw', 'None' if no flag was provided.156    """157 158    source: HfUri159    mount_path: str160    read_only: bool | None = None161    _raw: str | None = field(repr=False, hash=False, compare=False, default=None)162 163    def __post_init__(self) -> None:164        raw = self._raw or ""165        if not self.mount_path.startswith("/") or self.mount_path == "/":166            raise HfUriError(167                uri=raw,168                msg=f"Mount path must be a non-empty absolute path starting with '/', got '{self.mount_path}'.",169            )170 171    def to_uri(self) -> str:172        """Render the mount as a canonical 'hf://' string.173 174        Example: 'hf://models/my-org/my-model:/data:ro'175        """176        parts = [self.source.to_uri(), ":", self.mount_path]177        if self.read_only is not None:178            parts.append(":ro" if self.read_only else ":rw")179        return "".join(parts)180 181 182def parse_hf_uri(uri: str) -> HfUri:183    """Parse a Hugging Face Hub URI ('hf://...').184 185    A HF URI is a URI-like string identifying a location on the Hugging Face Hub. The full grammar is:186 187    ```188    hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]189    ```190 191    See 'docs/source/en/package_reference/hf_uris.md' for the full specification.192 193    Args:194        uri (`str`):195            The URI to parse. Must start with 'hf://'.196 197    Returns:198        [`HfUri`]: the parsed URI.199 200    Raises:201        [`HfUriError`]:202            If the URI is malformed (missing prefix, invalid type, missing id, etc.).203 204    Examples:205        ```py206        >>> from huggingface_hub.utils import parse_hf_uri207        >>> parse_hf_uri("hf://my-org/my-model")208        HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo='')209        >>> parse_hf_uri("hf://datasets/my-org/my-dataset@refs/pr/3/train.json")210        HfUri(type='dataset', id='my-org/my-dataset', revision='refs/pr/3', path_in_repo='train.json')211        ```212    """213    if not uri.startswith(constants.HF_PROTOCOL):214        raise HfUriError(215            uri,216            f"Must start with '{constants.HF_PROTOCOL}'. "217            f"Expected format: {constants.HF_PROTOCOL}[<TYPE>/]<ID>[@<REVISION>][/<PATH>]",218        )219 220    raw = uri221    body = uri[len(constants.HF_PROTOCOL) :]222    if not body:223        raise HfUriError(uri, f"Empty body after '{constants.HF_PROTOCOL}'.")224 225    type_, location = _split_type(body, raw=raw)226 227    if type_ == "bucket":228        return _parse_bucket_body(location, type_, raw=raw)229    return _parse_repo_body(location, type_, raw=raw)230 231 232def parse_hf_mount(mount_str: str) -> HfMount:233    """Parse a HF mount specification ('hf://...:<MOUNT_PATH>[:ro|:rw]').234 235    A mount specification is a HF URI followed by a local mount path and an optional read-only/read-write flag.236    The full grammar is:237 238    ```239    hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]240    ```241 242    See 'docs/source/en/package_reference/hf_uris.md' for the full specification.243 244    Args:245        mount_str (`str`):246            The mount string to parse. Must start with 'hf://' and contain a ':<MOUNT_PATH>' segment.247 248    Returns:249        [`HfMount`]: the parsed mount.250 251    Raises:252        [`HfUriError`]:253            If the mount string is malformed (missing mount path, invalid URI, etc.).254 255    Examples:256        ```py257        >>> from huggingface_hub.utils import parse_hf_mount258        >>> parse_hf_mount("hf://my-org/my-model:/data:ro")259        HfMount(source=HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo=''), mount_path='/data', read_only=True)260        >>> parse_hf_mount("hf://buckets/my-org/my-bucket/sub/dir:/mnt:rw")261        HfMount(source=HfUri(type='bucket', id='my-org/my-bucket', revision=None, path_in_repo='sub/dir'), mount_path='/mnt', read_only=False)262        ```263    """264    if not mount_str.startswith(constants.HF_PROTOCOL):265        raise HfUriError(266            uri=mount_str,267            msg=f"Must start with '{constants.HF_PROTOCOL}'.",268        )269 270    raw = mount_str271    body = mount_str[len(constants.HF_PROTOCOL) :]272    if not body:273        raise HfUriError(uri=raw, msg=f"Empty body after '{constants.HF_PROTOCOL}'.")274 275    location, mount_path, read_only = _split_mount(body, raw=raw)276 277    if mount_path is None:278        raise HfUriError(uri=raw, msg="Missing mount path. Expected ':<MOUNT_PATH>' (e.g. 'hf://org/model:/data').")279 280    # Re-assemble the URI part and parse it281    uri_str = constants.HF_PROTOCOL + location282    try:283        source = parse_hf_uri(uri_str)284    except HfUriError as e:285        raise HfUriError(uri=raw, msg=e.msg) from e286 287    return HfMount(source=source, mount_path=mount_path, read_only=read_only, _raw=raw)288 289 290def _split_mount(body: str, *, raw: str) -> tuple[str, str | None, bool | None]:291    """Split the ':<MOUNT_PATH>[:ro|:rw]' suffix from 'body'.292 293    Returns '(location, mount_path, read_only)' where 'mount_path' is 'None' if no mount segment is present.294    """295    if body.endswith(":ro"):296        read_only, body = True, body.removesuffix(":ro")297    elif body.endswith(":rw"):298        read_only, body = False, body.removesuffix(":rw")299    else:300        read_only = None301 302    # Mount paths always start with '/', so the delimiter is ':/'.303    # We use rfind() because the mount segment is always trailing304    idx = body.rfind(":/")305    if idx == -1:306        if read_only is not None:307            raise HfUriError(308                uri=raw,309                msg="':ro'/':rw' suffix is only valid when a mount path is provided (e.g. 'hf://...:/<MOUNT_PATH>:ro').",310            )311        return body, None, None312 313    location = body[:idx]314    mount_path = body[idx + 1 :]  # includes the leading '/'315    if not location:316        raise HfUriError(uri=raw, msg="Missing location before mount path.")317    return location, mount_path, read_only318 319 320def _split_type(location: str, *, raw: str) -> tuple[constants.HfUriType, str]:321    """Detect the (optional) type prefix and return '(type, remaining_location)'.322 323    A missing type prefix defaults to 'model'. Singular forms ('model/', 'dataset/', etc.) are explicitly rejected with a helpful error.324    """325    slash_idx = location.find("/")326    if slash_idx == -1:327        # Single segment, no prefix. Reject if it looks like a bare type name.328        if location in constants.HF_URI_TYPE_PREFIXES:329            raise HfUriError(330                uri=raw,331                msg=f"Missing identifier after '{location}'. Expected '{constants.HF_PROTOCOL}{location}/<ID>'.",332            )333        if (singular_plural := _TYPE_TO_PREFIX.get(location)) is not None:334            raise HfUriError(335                uri=raw,336                msg=f"Type prefix must be plural. Did you mean '{constants.HF_PROTOCOL}{singular_plural}/...'?",337            )338        return "model", location339 340    first = location[:slash_idx]341    rest = location[slash_idx + 1 :]342    if first in constants.HF_URI_TYPE_PREFIXES:343        return constants.HF_URI_TYPE_PREFIXES[first], rest344    if (singular_plural := _TYPE_TO_PREFIX.get(first)) is not None:345        raise HfUriError(346            uri=raw, msg=f"Type prefix must be plural, got '{first}/'. Did you mean '{singular_plural}/'?"347        )348    return "model", location349 350 351def _parse_bucket_body(352    location: str,353    type_: constants.HfUriType,354    *,355    raw: str,356) -> HfUri:357    """Parse the body of a bucket URI: 'namespace/name[/path]'."""358    if "@" in location:359        raise HfUriError(uri=raw, msg="Bucket URIs do not support a revision marker ('@').")360    location = location.strip("/")361    parts = location.split("/", 2)362    if len(parts) < 2 or not parts[0] or not parts[1]:363        raise HfUriError(uri=raw, msg=f"Bucket id must be 'namespace/name', got '{location}'.")364    bucket_id = f"{parts[0]}/{parts[1]}"365    path_in_bucket = parts[2] if len(parts) >= 3 else ""366    return HfUri(367        type=type_,368        id=bucket_id,369        revision=None,370        path_in_repo=path_in_bucket,371        _raw=raw,372    )373 374 375def _parse_repo_body(376    location: str,377    type_: constants.HfUriType,378    *,379    raw: str,380) -> HfUri:381    """Parse the body of a repo URI: '<repo_id>[@<revision>][/<path>]'."""382    location = location.strip("/")383    if not location:384        raise HfUriError(uri=raw, msg="Missing repository id.")385 386    # The first '@' separates the repo_id from the revision (and rest of path).387    # No valid repo_id contains '@' and no valid revision contains '@'.388    at_idx = location.find("@")389    revision: str | None390    if at_idx == -1:391        # No revision. Take the first 2 segments as repo_id, rest as path_in_repo.392        revision = None393        parts = location.split("/", 2)394        if len(parts) < 2:395            raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{location}'. ")396        repo_id = f"{parts[0]}/{parts[1]}"397        path_in_repo = parts[2] if len(parts) > 2 else ""398    else:399        repo_id = location[:at_idx]400        rev_and_path = location[at_idx + 1 :]401        if not repo_id:402            raise HfUriError(uri=raw, msg="Missing repository id before '@'.")403        if repo_id.count("/") != 1:404            raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{repo_id}'.")405        # Special refs like 'refs/pr/10' contain '/' and must be matched eagerly,406        # otherwise we would split them at the first '/' and treat the rest as a path.407        match = _SPECIAL_REFS_REVISION_REGEX.match(rev_and_path)408        if match is not None:409            revision = match.group()410            path_in_repo = rev_and_path[len(revision) :].removeprefix("/")411        else:412            slash_idx = rev_and_path.find("/")413            if slash_idx == -1:414                revision = rev_and_path415                path_in_repo = ""416            else:417                revision = rev_and_path[:slash_idx]418                path_in_repo = rev_and_path[slash_idx + 1 :]419        revision = unquote(revision)420        if not revision:421            raise HfUriError(uri=raw, msg="Empty revision after '@'.")422 423    return HfUri(424        type=type_,425        id=repo_id,426        revision=revision,427        path_in_repo=path_in_repo,428        _raw=raw,429    )430