codekingpro/portable-devtools

1"""URL parsing utilities."""2 3import re4import unicodedata5from functools import lru_cache6from urllib.parse import scheme_chars, uses_netloc7 8from ._quoters import QUOTER, UNQUOTER_PLUS9 10# Leading and trailing C0 control and space to be stripped per WHATWG spec.11# == "".join([chr(i) for i in range(0, 0x20 + 1)])12WHATWG_C0_CONTROL_OR_SPACE = (13    "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10"14    "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f "15)16 17# Unsafe bytes to be removed per WHATWG spec18UNSAFE_URL_BYTES_TO_REMOVE = ["\t", "\r", "\n"]19USES_AUTHORITY = frozenset(uses_netloc)20 21SplitURLType = tuple[str, str, str, str, str]22 23 24def split_url(url: str) -> SplitURLType:25    """Split URL into parts."""26    # Adapted from urllib.parse.urlsplit27    # Only lstrip url as some applications rely on preserving trailing space.28    # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both)29    url = url.lstrip(WHATWG_C0_CONTROL_OR_SPACE)30    for b in UNSAFE_URL_BYTES_TO_REMOVE:31        if b in url:32            url = url.replace(b, "")33 34    scheme = netloc = query = fragment = ""35    i = url.find(":")36    if i > 0 and url[0] in scheme_chars:37        for c in url[1:i]:38            if c not in scheme_chars:39                break40        else:41            scheme, url = url[:i].lower(), url[i + 1 :]42    has_hash = "#" in url43    has_question_mark = "?" in url44    if url[:2] == "//":45        delim = len(url)  # position of end of domain part of url, default is end46        if has_hash and has_question_mark:47            delim_chars = "/?#"48        elif has_question_mark:49            delim_chars = "/?"50        elif has_hash:51            delim_chars = "/#"52        else:53            delim_chars = "/"54        for c in delim_chars:  # look for delimiters; the order is NOT important55            wdelim = url.find(c, 2)  # find first of this delim56            if wdelim >= 0 and wdelim < delim:  # if found57                delim = wdelim  # use earliest delim position58        netloc = url[2:delim]59        url = url[delim:]60        has_left_bracket = "[" in netloc61        has_right_bracket = "]" in netloc62        if (has_left_bracket and not has_right_bracket) or (63            has_right_bracket and not has_left_bracket64        ):65            raise ValueError("Invalid IPv6 URL")66        if has_left_bracket:67            bracketed_host = netloc.partition("[")[2].partition("]")[0]68            # Valid bracketed hosts are defined in69            # https://www.rfc-editor.org/rfc/rfc3986#page-4970            # https://url.spec.whatwg.org/71            if bracketed_host and bracketed_host[0] == "v":72                if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", bracketed_host):73                    raise ValueError("IPvFuture address is invalid")74            elif ":" not in bracketed_host:75                raise ValueError("The IPv6 content between brackets is not valid")76    if has_hash:77        url, _, fragment = url.partition("#")78    if has_question_mark:79        url, _, query = url.partition("?")80    if netloc and not netloc.isascii():81        _check_netloc(netloc)82    return scheme, netloc, url, query, fragment83 84 85def _check_netloc(netloc: str) -> None:86    # Adapted from urllib.parse._checknetloc87    # looking for characters like \u2100 that expand to 'a/c'88    # IDNA uses NFKC equivalence, so normalize for this check89 90    # ignore characters already included91    # but not the surrounding text92    n = netloc.replace("@", "").replace(":", "").replace("#", "").replace("?", "")93    normalized_netloc = unicodedata.normalize("NFKC", n)94    if n == normalized_netloc:95        return96    # Note that there are no unicode decompositions for the character '@' so97    # its currently impossible to have test coverage for this branch, however if the98    # one should be added in the future we want to make sure its still checked.99    for c in "/?#@:":  # pragma: no branch100        if c in normalized_netloc:101            raise ValueError(102                f"netloc '{netloc}' contains invalid "103                "characters under NFKC normalization"104            )105 106 107@lru_cache  # match the same size as urlsplit108def split_netloc(109    netloc: str,110) -> tuple[str | None, str | None, str | None, int | None]:111    """Split netloc into username, password, host and port."""112    if "@" not in netloc:113        username: str | None = None114        password: str | None = None115        hostinfo = netloc116    else:117        userinfo, _, hostinfo = netloc.rpartition("@")118        username, have_password, password = userinfo.partition(":")119        if not have_password:120            password = None121 122    if "[" in hostinfo:123        _, _, bracketed = hostinfo.partition("[")124        hostname, _, port_str = bracketed.partition("]")125        _, _, port_str = port_str.partition(":")126    else:127        hostname, _, port_str = hostinfo.partition(":")128 129    if not port_str:130        return username or None, password, hostname or None, None131 132    try:133        port = int(port_str)134    except ValueError:135        raise ValueError("Invalid URL: port can't be converted to integer")136    if not (0 <= port <= 65535):137        raise ValueError("Port out of range 0-65535")138    return username or None, password, hostname or None, port139 140 141def unsplit_result(142    scheme: str, netloc: str, url: str, query: str, fragment: str143) -> str:144    """Unsplit a URL without any normalization."""145    if netloc or (scheme and scheme in USES_AUTHORITY) or url[:2] == "//":146        if url and url[:1] != "/":147            url = f"{scheme}://{netloc}/{url}" if scheme else f"{scheme}:{url}"148        else:149            url = f"{scheme}://{netloc}{url}" if scheme else f"//{netloc}{url}"150    elif scheme:151        url = f"{scheme}:{url}"152    if query:153        url = f"{url}?{query}"154    return f"{url}#{fragment}" if fragment else url155 156 157@lru_cache  # match the same size as urlsplit158def make_netloc(159    user: str | None,160    password: str | None,161    host: str | None,162    port: int | None,163    encode: bool = False,164) -> str:165    """Make netloc from parts.166 167    The user and password are encoded if encode is True.168 169    The host must already be encoded with _encode_host.170    """171    if host is None:172        return ""173    ret = host174    if port is not None:175        ret = f"{ret}:{port}"176    if user is None and password is None:177        return ret178    if password is not None:179        if not user:180            user = ""181        elif encode:182            user = QUOTER(user)183        if encode:184            password = QUOTER(password)185        user = f"{user}:{password}"186    elif user and encode:187        user = QUOTER(user)188    return f"{user}@{ret}" if user else ret189 190 191def query_to_pairs(query_string: str) -> list[tuple[str, str]]:192    """Parse a query given as a string argument.193 194    Works like urllib.parse.parse_qsl with keep empty values.195    """196    pairs: list[tuple[str, str]] = []197    if not query_string:198        return pairs199    for k_v in query_string.split("&"):200        k, _, v = k_v.partition("=")201        pairs.append((UNQUOTER_PLUS(k), UNQUOTER_PLUS(v)))202    return pairs203