codekingpro/portable-devtools
114k
1import re2import sys3import warnings4from collections.abc import Mapping, Sequence5from enum import Enum6from functools import _CacheInfo, lru_cache7from ipaddress import ip_address8from typing import (9 TYPE_CHECKING,10 Any,11 NoReturn,12 TypedDict,13 TypeVar,14 Union,15 cast,16 overload,17)18from urllib.parse import SplitResult, uses_relative19 20import idna21from multidict import MultiDict, MultiDictProxy, istr22from propcache.api import under_cached_property as cached_property23 24from ._parse import (25 USES_AUTHORITY,26 SplitURLType,27 make_netloc,28 query_to_pairs,29 split_netloc,30 split_url,31 unsplit_result,32)33from ._path import normalize_path, normalize_path_segments34from ._query import (35 Query,36 QueryVariable,37 SimpleQuery,38 get_str_query,39 get_str_query_from_iterable,40 get_str_query_from_sequence_iterable,41)42from ._quoters import (43 FRAGMENT_QUOTER,44 FRAGMENT_REQUOTER,45 PATH_QUOTER,46 PATH_REQUOTER,47 PATH_SAFE_UNQUOTER,48 PATH_UNQUOTER,49 QS_UNQUOTER,50 QUERY_QUOTER,51 QUERY_REQUOTER,52 QUOTER,53 REQUOTER,54 UNQUOTER,55 human_quote,56)57 58try:59 from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler60 from pydantic.json_schema import JsonSchemaValue61 from pydantic_core import core_schema62 63 HAS_PYDANTIC = True64except ImportError:65 HAS_PYDANTIC = False66 67 68DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}69USES_RELATIVE = frozenset(uses_relative)70 71# Special schemes https://url.spec.whatwg.org/#special-scheme72# are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation73SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp"))74 75 76# reg-name: unreserved / pct-encoded / sub-delims77# this pattern matches anything that is *not* in those classes. and is only used78# on lower-cased ASCII values.79NOT_REG_NAME = re.compile(80 r"""81 # any character not in the unreserved or sub-delims sets, plus %82 # (validated with the additional check for pct-encoded sequences below)83 [^a-z0-9\-._~!$&'()*+,;=%]84 |85 # % only allowed if it is part of a pct-encoded86 # sequence of 2 hex digits.87 %(?![0-9a-f]{2})88 """,89 re.VERBOSE,90)91 92_T = TypeVar("_T")93 94if sys.version_info >= (3, 11):95 from typing import Self96else:97 Self = Any98 99 100class UndefinedType(Enum):101 """Singleton type for use with not set sentinel values."""102 103 _singleton = 0104 105 106UNDEFINED = UndefinedType._singleton107 108 109class CacheInfo(TypedDict):110 """Host encoding cache."""111 112 idna_encode: _CacheInfo113 idna_decode: _CacheInfo114 ip_address: _CacheInfo115 host_validate: _CacheInfo116 encode_host: _CacheInfo117 118 119class _InternalURLCache(TypedDict, total=False):120 _val: SplitURLType121 _origin: "URL"122 absolute: bool123 hash: int124 scheme: str125 raw_authority: str126 authority: str127 raw_user: str | None128 user: str | None129 raw_password: str | None130 password: str | None131 raw_host: str | None132 host: str | None133 host_subcomponent: str | None134 host_port_subcomponent: str | None135 port: int | None136 explicit_port: int | None137 raw_path: str138 path: str139 _parsed_query: list[tuple[str, str]]140 query: "MultiDictProxy[str]"141 raw_query_string: str142 query_string: str143 path_qs: str144 raw_path_qs: str145 raw_fragment: str146 fragment: str147 raw_parts: tuple[str, ...]148 parts: tuple[str, ...]149 parent: "URL"150 raw_name: str151 name: str152 raw_suffix: str153 suffix: str154 raw_suffixes: tuple[str, ...]155 suffixes: tuple[str, ...]156 157 158def rewrite_module(obj: _T) -> _T:159 obj.__module__ = "yarl"160 return obj161 162 163@lru_cache164def encode_url(url_str: str) -> "URL":165 """Parse unencoded URL."""166 cache: _InternalURLCache = {}167 host: str | None168 scheme, netloc, path, query, fragment = split_url(url_str)169 if not netloc: # netloc170 host = ""171 else:172 if ":" in netloc or "@" in netloc or "[" in netloc:173 # Complex netloc174 username, password, host, port = split_netloc(netloc)175 else:176 username = password = port = None177 host = netloc178 if host is None:179 if scheme in SCHEME_REQUIRES_HOST:180 msg = (181 "Invalid URL: host is required for "182 f"absolute urls with the {scheme} scheme"183 )184 raise ValueError(msg)185 else:186 host = ""187 host = _encode_host(host, validate_host=False)188 # Remove brackets as host encoder adds back brackets for IPv6 addresses189 cache["raw_host"] = host[1:-1] if "[" in host else host190 cache["explicit_port"] = port191 if password is None and username is None:192 # Fast path for URLs without user, password193 netloc = host if port is None else f"{host}:{port}"194 cache["raw_user"] = None195 cache["raw_password"] = None196 else:197 raw_user = REQUOTER(username) if username else username198 raw_password = REQUOTER(password) if password else password199 netloc = make_netloc(raw_user, raw_password, host, port)200 cache["raw_user"] = raw_user201 cache["raw_password"] = raw_password202 203 if path:204 path = PATH_REQUOTER(path)205 if netloc and "." in path:206 path = normalize_path(path)207 if query:208 query = QUERY_REQUOTER(query)209 if fragment:210 fragment = FRAGMENT_REQUOTER(fragment)211 212 cache["scheme"] = scheme213 cache["raw_path"] = "/" if not path and netloc else path214 cache["raw_query_string"] = query215 cache["raw_fragment"] = fragment216 217 self = object.__new__(URL)218 self._scheme = scheme219 self._netloc = netloc220 self._path = path221 self._query = query222 self._fragment = fragment223 self._cache = cache224 return self225 226 227@lru_cache228def pre_encoded_url(url_str: str) -> "URL":229 """Parse pre-encoded URL."""230 self = object.__new__(URL)231 val = split_url(url_str)232 self._scheme, self._netloc, self._path, self._query, self._fragment = val233 self._cache = {}234 return self235 236 237@lru_cache238def build_pre_encoded_url(239 scheme: str,240 authority: str,241 user: str | None,242 password: str | None,243 host: str,244 port: int | None,245 path: str,246 query_string: str,247 fragment: str,248) -> "URL":249 """Build a pre-encoded URL from parts."""250 self = object.__new__(URL)251 self._scheme = scheme252 if authority:253 self._netloc = authority254 elif host:255 if port is not None:256 port = None if port == DEFAULT_PORTS.get(scheme) else port257 if user is None and password is None:258 self._netloc = host if port is None else f"{host}:{port}"259 else:260 self._netloc = make_netloc(user, password, host, port)261 else:262 self._netloc = ""263 self._path = path264 self._query = query_string265 self._fragment = fragment266 self._cache = {}267 return self268 269 270def from_parts_uncached(271 scheme: str, netloc: str, path: str, query: str, fragment: str272) -> "URL":273 """Create a new URL from parts."""274 self = object.__new__(URL)275 self._scheme = scheme276 self._netloc = netloc277 self._path = path278 self._query = query279 self._fragment = fragment280 self._cache = {}281 return self282 283 284from_parts = lru_cache(from_parts_uncached)285 286 287@rewrite_module288class URL:289 # Don't derive from str290 # follow pathlib.Path design291 # probably URL will not suffer from pathlib problems:292 # it's intended for libraries like aiohttp,293 # not to be passed into standard library functions like os.open etc.294 295 # URL grammar (RFC 3986)296 # pct-encoded = "%" HEXDIG HEXDIG297 # reserved = gen-delims / sub-delims298 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"299 # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"300 # / "*" / "+" / "," / ";" / "="301 # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"302 # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]303 # hier-part = "//" authority path-abempty304 # / path-absolute305 # / path-rootless306 # / path-empty307 # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )308 # authority = [ userinfo "@" ] host [ ":" port ]309 # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )310 # host = IP-literal / IPv4address / reg-name311 # IP-literal = "[" ( IPv6address / IPvFuture ) "]"312 # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )313 # IPv6address = 6( h16 ":" ) ls32314 # / "::" 5( h16 ":" ) ls32315 # / [ h16 ] "::" 4( h16 ":" ) ls32316 # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32317 # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32318 # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32319 # / [ *4( h16 ":" ) h16 ] "::" ls32320 # / [ *5( h16 ":" ) h16 ] "::" h16321 # / [ *6( h16 ":" ) h16 ] "::"322 # ls32 = ( h16 ":" h16 ) / IPv4address323 # ; least-significant 32 bits of address324 # h16 = 1*4HEXDIG325 # ; 16 bits of address represented in hexadecimal326 # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet327 # dec-octet = DIGIT ; 0-9328 # / %x31-39 DIGIT ; 10-99329 # / "1" 2DIGIT ; 100-199330 # / "2" %x30-34 DIGIT ; 200-249331 # / "25" %x30-35 ; 250-255332 # reg-name = *( unreserved / pct-encoded / sub-delims )333 # port = *DIGIT334 # path = path-abempty ; begins with "/" or is empty335 # / path-absolute ; begins with "/" but not "//"336 # / path-noscheme ; begins with a non-colon segment337 # / path-rootless ; begins with a segment338 # / path-empty ; zero characters339 # path-abempty = *( "/" segment )340 # path-absolute = "/" [ segment-nz *( "/" segment ) ]341 # path-noscheme = segment-nz-nc *( "/" segment )342 # path-rootless = segment-nz *( "/" segment )343 # path-empty = 0<pchar>344 # segment = *pchar345 # segment-nz = 1*pchar346 # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )347 # ; non-zero-length segment without any colon ":"348 # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"349 # query = *( pchar / "/" / "?" )350 # fragment = *( pchar / "/" / "?" )351 # URI-reference = URI / relative-ref352 # relative-ref = relative-part [ "?" query ] [ "#" fragment ]353 # relative-part = "//" authority path-abempty354 # / path-absolute355 # / path-noscheme356 # / path-empty357 # absolute-URI = scheme ":" hier-part [ "?" query ]358 __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment")359 360 _cache: _InternalURLCache361 _scheme: str362 _netloc: str363 _path: str364 _query: str365 _fragment: str366 367 def __new__(368 cls,369 val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED,370 *,371 encoded: bool = False,372 strict: bool | None = None,373 ) -> "URL":374 if strict is not None: # pragma: no cover375 warnings.warn("strict parameter is ignored")376 if type(val) is str:377 return pre_encoded_url(val) if encoded else encode_url(val)378 if type(val) is cls:379 return val380 if type(val) is SplitResult:381 if not encoded:382 raise ValueError("Cannot apply decoding to SplitResult")383 return from_parts(*val)384 if isinstance(val, str):385 return pre_encoded_url(str(val)) if encoded else encode_url(str(val))386 if val is UNDEFINED:387 # Special case for UNDEFINED since it might be unpickling and we do388 # not want to cache as the `__set_state__` call would mutate the URL389 # object in the `pre_encoded_url` or `encoded_url` caches.390 self = object.__new__(URL)391 self._scheme = self._netloc = self._path = self._query = self._fragment = ""392 self._cache = {}393 return self394 raise TypeError("Constructor parameter should be str")395 396 @classmethod397 def build(398 cls,399 *,400 scheme: str = "",401 authority: str = "",402 user: str | None = None,403 password: str | None = None,404 host: str = "",405 port: int | None = None,406 path: str = "",407 query: Query | None = None,408 query_string: str = "",409 fragment: str = "",410 encoded: bool = False,411 ) -> "URL":412 """Creates and returns a new URL"""413 414 if authority and (user or password or host or port):415 raise ValueError(416 'Can\'t mix "authority" with "user", "password", "host" or "port".'417 )418 if port is not None and not isinstance(port, int):419 raise TypeError(f"The port is required to be int, got {type(port)!r}.")420 if port and not host:421 raise ValueError('Can\'t build URL with "port" but without "host".')422 if query and query_string:423 raise ValueError('Only one of "query" or "query_string" should be passed')424 if (425 scheme is None # type: ignore[redundant-expr]426 or authority is None # type: ignore[redundant-expr]427 or host is None # type: ignore[redundant-expr]428 or path is None # type: ignore[redundant-expr]429 or query_string is None # type: ignore[redundant-expr]430 or fragment is None431 ):432 raise TypeError(433 'NoneType is illegal for "scheme", "authority", "host", "path", '434 '"query_string", and "fragment" args, use empty string instead.'435 )436 437 if query:438 query_string = get_str_query(query) or ""439 440 if encoded:441 return build_pre_encoded_url(442 scheme,443 authority,444 user,445 password,446 host,447 port,448 path,449 query_string,450 fragment,451 )452 453 self = object.__new__(URL)454 self._scheme = scheme455 _host: str | None = None456 if authority:457 user, password, _host, port = split_netloc(authority)458 _host = _encode_host(_host, validate_host=False) if _host else ""459 elif host:460 _host = _encode_host(host, validate_host=True)461 else:462 self._netloc = ""463 464 if _host is not None:465 if port is not None:466 port = None if port == DEFAULT_PORTS.get(scheme) else port467 if user is None and password is None:468 self._netloc = _host if port is None else f"{_host}:{port}"469 else:470 self._netloc = make_netloc(user, password, _host, port, True)471 472 path = PATH_QUOTER(path) if path else path473 if path and self._netloc:474 if "." in path:475 path = normalize_path(path)476 if path[0] != "/":477 msg = (478 "Path in a URL with authority should "479 "start with a slash ('/') if set"480 )481 raise ValueError(msg)482 483 self._path = path484 if not query and query_string:485 query_string = QUERY_QUOTER(query_string)486 self._query = query_string487 self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment488 self._cache = {}489 return self490 491 def __init_subclass__(cls) -> NoReturn:492 raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden")493 494 def __str__(self) -> str:495 if not self._path and self._netloc and (self._query or self._fragment):496 path = "/"497 else:498 path = self._path499 if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(500 self._scheme501 ):502 # port normalization - using None for default ports to remove from rendering503 # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3504 host = self.host_subcomponent505 netloc = make_netloc(self.raw_user, self.raw_password, host, None)506 else:507 netloc = self._netloc508 return unsplit_result(self._scheme, netloc, path, self._query, self._fragment)509 510 def __repr__(self) -> str:511 return f"{self.__class__.__name__}('{str(self)}')"512 513 def __bytes__(self) -> bytes:514 return str(self).encode("ascii")515 516 def __eq__(self, other: object) -> bool:517 if type(other) is not URL:518 return NotImplemented519 520 path1 = "/" if not self._path and self._netloc else self._path521 path2 = "/" if not other._path and other._netloc else other._path522 return (523 self._scheme == other._scheme524 and self._netloc == other._netloc525 and path1 == path2526 and self._query == other._query527 and self._fragment == other._fragment528 )529 530 def __hash__(self) -> int:531 if (ret := self._cache.get("hash")) is None:532 path = "/" if not self._path and self._netloc else self._path533 ret = self._cache["hash"] = hash(534 (self._scheme, self._netloc, path, self._query, self._fragment)535 )536 return ret537 538 def __le__(self, other: object) -> bool:539 if type(other) is not URL:540 return NotImplemented541 return self._val <= other._val542 543 def __lt__(self, other: object) -> bool:544 if type(other) is not URL:545 return NotImplemented546 return self._val < other._val547 548 def __ge__(self, other: object) -> bool:549 if type(other) is not URL:550 return NotImplemented551 return self._val >= other._val552 553 def __gt__(self, other: object) -> bool:554 if type(other) is not URL:555 return NotImplemented556 return self._val > other._val557 558 def __truediv__(self, name: str) -> "URL":559 if not isinstance(name, str):560 return NotImplemented # type: ignore[unreachable]561 return self._make_child((str(name),))562 563 def __mod__(self, query: Query) -> "URL":564 return self.update_query(query)565 566 def __bool__(self) -> bool:567 return bool(self._netloc or self._path or self._query or self._fragment)568 569 def __getstate__(self) -> tuple[SplitResult]:570 return (tuple.__new__(SplitResult, self._val),)571 572 def __setstate__(573 self, state: tuple[SplitURLType] | tuple[None, _InternalURLCache]574 ) -> None:575 if state[0] is None and isinstance(state[1], dict):576 # default style pickle577 val = state[1]["_val"]578 else:579 unused: list[object]580 val, *unused = state581 self._scheme, self._netloc, self._path, self._query, self._fragment = val582 self._cache = {}583 584 def _cache_netloc(self) -> None:585 """Cache the netloc parts of the URL."""586 c = self._cache587 split_loc = split_netloc(self._netloc)588 c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc589 590 def is_absolute(self) -> bool:591 """A check for absolute URLs.592 593 Return True for absolute ones (having scheme or starting594 with //), False otherwise.595 596 Is is preferred to call the .absolute property instead597 as it is cached.598 """599 return self.absolute600 601 def is_default_port(self) -> bool:602 """A check for default port.603 604 Return True if port is default for specified scheme,605 e.g. 'http://python.org' or 'http://python.org:80', False606 otherwise.607 608 Return False for relative URLs.609 610 """611 if (explicit := self.explicit_port) is None:612 # If the explicit port is None, then the URL must be613 # using the default port unless its a relative URL614 # which does not have an implicit port / default port615 return self._netloc != ""616 return explicit == DEFAULT_PORTS.get(self._scheme)617 618 def origin(self) -> "URL":619 """Return an URL with scheme, host and port parts only.620 621 user, password, path, query and fragment are removed.622 623 """624 # TODO: add a keyword-only option for keeping user/pass maybe?625 return self._origin626 627 @cached_property628 def _val(self) -> SplitURLType:629 return (self._scheme, self._netloc, self._path, self._query, self._fragment)630 631 @cached_property632 def _origin(self) -> "URL":633 """Return an URL with scheme, host and port parts only.634 635 user, password, path, query and fragment are removed.636 """637 if not (netloc := self._netloc):638 raise ValueError("URL should be absolute")639 if not (scheme := self._scheme):640 raise ValueError("URL should have scheme")641 if "@" in netloc:642 encoded_host = self.host_subcomponent643 netloc = make_netloc(None, None, encoded_host, self.explicit_port)644 elif not self._path and not self._query and not self._fragment:645 return self646 return from_parts(scheme, netloc, "", "", "")647 648 def relative(self) -> "URL":649 """Return a relative part of the URL.650 651 scheme, user, password, host and port are removed.652 653 """654 if not self._netloc:655 raise ValueError("URL should be absolute")656 return from_parts("", "", self._path, self._query, self._fragment)657 658 @cached_property659 def absolute(self) -> bool:660 """A check for absolute URLs.661 662 Return True for absolute ones (having scheme or starting663 with //), False otherwise.664 665 """666 # `netloc`` is an empty string for relative URLs667 # Checking `netloc` is faster than checking `hostname`668 # because `hostname` is a property that does some extra work669 # to parse the host from the `netloc`670 return self._netloc != ""671 672 @cached_property673 def scheme(self) -> str:674 """Scheme for absolute URLs.675 676 Empty string for relative URLs or URLs starting with //677 678 """679 return self._scheme680 681 @cached_property682 def raw_authority(self) -> str:683 """Encoded authority part of URL.684 685 Empty string for relative URLs.686 687 """688 return self._netloc689 690 @cached_property691 def authority(self) -> str:692 """Decoded authority part of URL.693 694 Empty string for relative URLs.695 696 """697 return make_netloc(self.user, self.password, self.host, self.port)698 699 @cached_property700 def raw_user(self) -> str | None:701 """Encoded user part of URL.702 703 None if user is missing.704 705 """706 # not .username707 self._cache_netloc()708 return self._cache["raw_user"]709 710 @cached_property711 def user(self) -> str | None:712 """Decoded user part of URL.713 714 None if user is missing.715 716 """717 if (raw_user := self.raw_user) is None:718 return None719 return UNQUOTER(raw_user)720 721 @cached_property722 def raw_password(self) -> str | None:723 """Encoded password part of URL.724 725 None if password is missing.726 727 """728 self._cache_netloc()729 return self._cache["raw_password"]730 731 @cached_property732 def password(self) -> str | None:733 """Decoded password part of URL.734 735 None if password is missing.736 737 """738 if (raw_password := self.raw_password) is None:739 return None740 return UNQUOTER(raw_password)741 742 @cached_property743 def raw_host(self) -> str | None:744 """Encoded host part of URL.745 746 None for relative URLs.747 748 When working with IPv6 addresses, use the `host_subcomponent` property instead749 as it will return the host subcomponent with brackets.750 """751 # Use host instead of hostname for sake of shortness752 # May add .hostname prop later753 self._cache_netloc()754 return self._cache["raw_host"]755 756 @cached_property757 def host(self) -> str | None:758 """Decoded host part of URL.759 760 None for relative URLs.761 762 """763 if (raw := self.raw_host) is None:764 return None765 if raw and raw[-1].isdigit() or ":" in raw:766 # IP addresses are never IDNA encoded767 return raw768 return _idna_decode(raw)769 770 @cached_property771 def host_subcomponent(self) -> str | None:772 """Return the host subcomponent part of URL.773 774 None for relative URLs.775 776 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2777 778 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`779 780 Examples:781 - `http://example.com:8080` -> `example.com`782 - `http://example.com:80` -> `example.com`783 - `https://127.0.0.1:8443` -> `127.0.0.1`784 - `https://[::1]:8443` -> `[::1]`785 - `http://[::1]` -> `[::1]`786 787 """788 if (raw := self.raw_host) is None:789 return None790 return f"[{raw}]" if ":" in raw else raw791 792 @cached_property793 def host_port_subcomponent(self) -> str | None:794 """Return the host and port subcomponent part of URL.795 796 Trailing dots are removed from the host part.797 798 This value is suitable for use in the Host header of an HTTP request.799 800 None for relative URLs.801 802 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2803 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`804 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3805 port = *DIGIT806 807 Examples:808 - `http://example.com:8080` -> `example.com:8080`809 - `http://example.com:80` -> `example.com`810 - `http://example.com.:80` -> `example.com`811 - `https://127.0.0.1:8443` -> `127.0.0.1:8443`812 - `https://[::1]:8443` -> `[::1]:8443`813 - `http://[::1]` -> `[::1]`814 815 """816 if (raw := self.raw_host) is None:817 return None818 if raw[-1] == ".":819 # Remove all trailing dots from the netloc as while820 # they are valid FQDNs in DNS, TLS validation fails.821 # See https://github.com/aio-libs/aiohttp/issues/3636.822 # To avoid string manipulation we only call rstrip if823 # the last character is a dot.824 raw = raw.rstrip(".")825 port = self.explicit_port826 if port is None or port == DEFAULT_PORTS.get(self._scheme):827 return f"[{raw}]" if ":" in raw else raw828 return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}"829 830 @cached_property831 def port(self) -> int | None:832 """Port part of URL, with scheme-based fallback.833 834 None for relative URLs or URLs without explicit port and835 scheme without default port substitution.836 837 """838 if (explicit_port := self.explicit_port) is not None:839 return explicit_port840 return DEFAULT_PORTS.get(self._scheme)841 842 @cached_property843 def explicit_port(self) -> int | None:844 """Port part of URL, without scheme-based fallback.845 846 None for relative URLs or URLs without explicit port.847 848 """849 self._cache_netloc()850 return self._cache["explicit_port"]851 852 @cached_property853 def raw_path(self) -> str:854 """Encoded path of URL.855 856 / for absolute URLs without path part.857 858 """859 return self._path if self._path or not self._netloc else "/"860 861 @cached_property862 def path(self) -> str:863 """Decoded path of URL.864 865 / for absolute URLs without path part.866 867 """868 return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else ""869 870 @cached_property871 def path_safe(self) -> str:872 """Decoded path of URL.873 874 / for absolute URLs without path part.875 876 / (%2F) and % (%25) are not decoded877 878 """879 if self._path:880 return PATH_SAFE_UNQUOTER(self._path)881 return "/" if self._netloc else ""882 883 @cached_property884 def _parsed_query(self) -> list[tuple[str, str]]:885 """Parse query part of URL."""886 return query_to_pairs(self._query)887 888 @cached_property889 def query(self) -> "MultiDictProxy[str]":890 """A MultiDictProxy representing parsed query parameters in decoded891 representation.892 893 Empty value if URL has no query part.894 895 """896 return MultiDictProxy(MultiDict(self._parsed_query))897 898 @cached_property899 def raw_query_string(self) -> str:900 """Encoded query part of URL.901 902 Empty string if query is missing.903 904 """905 return self._query906 907 @cached_property908 def query_string(self) -> str:909 """Decoded query part of URL.910 911 Empty string if query is missing.912 913 """914 return QS_UNQUOTER(self._query) if self._query else ""915 916 @cached_property917 def path_qs(self) -> str:918 """Decoded path of URL with query."""919 return self.path if not (q := self.query_string) else f"{self.path}?{q}"920 921 @cached_property922 def raw_path_qs(self) -> str:923 """Encoded path of URL with query."""924 if q := self._query:925 return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}"926 return self._path if self._path or not self._netloc else "/"927 928 @cached_property929 def raw_fragment(self) -> str:930 """Encoded fragment part of URL.931 932 Empty string if fragment is missing.933 934 """935 return self._fragment936 937 @cached_property938 def fragment(self) -> str:939 """Decoded fragment part of URL.940 941 Empty string if fragment is missing.942 943 """944 return UNQUOTER(self._fragment) if self._fragment else ""945 946 @cached_property947 def raw_parts(self) -> tuple[str, ...]:948 """A tuple containing encoded *path* parts.949 950 ('/',) for absolute URLs if *path* is missing.951 952 """953 path = self._path954 if self._netloc:955 return ("/", *path[1:].split("/")) if path else ("/",)956 if path and path[0] == "/":957 return ("/", *path[1:].split("/"))958 return tuple(path.split("/"))959 960 @cached_property961 def parts(self) -> tuple[str, ...]:962 """A tuple containing decoded *path* parts.963 964 ('/',) for absolute URLs if *path* is missing.965 966 """967 return tuple(UNQUOTER(part) for part in self.raw_parts)968 969 @cached_property970 def parent(self) -> "URL":971 """A new URL with last part of path removed and cleaned up query and972 fragment.973 974 """975 path = self._path976 if not path or path == "/":977 if self._fragment or self._query:978 return from_parts(self._scheme, self._netloc, path, "", "")979 return self980 parts = path.split("/")981 return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "")982 983 @cached_property984 def raw_name(self) -> str:985 """The last part of raw_parts."""986 parts = self.raw_parts987 if not self._netloc:988 return parts[-1]989 parts = parts[1:]990 return parts[-1] if parts else ""991 992 @cached_property993 def name(self) -> str:994 """The last part of parts."""995 return UNQUOTER(self.raw_name)996 997 @cached_property998 def raw_suffix(self) -> str:999 name = self.raw_name1000 i = name.rfind(".")1001 return name[i:] if 0 < i < len(name) - 1 else ""1002 1003 @cached_property1004 def suffix(self) -> str:1005 return UNQUOTER(self.raw_suffix)1006 1007 @cached_property1008 def raw_suffixes(self) -> tuple[str, ...]:1009 name = self.raw_name1010 if name.endswith("."):1011 return ()1012 name = name.lstrip(".")1013 return tuple("." + suffix for suffix in name.split(".")[1:])1014 1015 @cached_property1016 def suffixes(self) -> tuple[str, ...]:1017 return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes)1018 1019 def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL":1020 """1021 add paths to self._path, accounting for absolute vs relative paths,1022 keep existing, but do not create new, empty segments1023 """1024 parsed: list[str] = []1025 needs_normalize: bool = False1026 for idx, path in enumerate(reversed(paths)):1027 # empty segment of last is not removed1028 last = idx == 01029 if path and path[0] == "/":1030 raise ValueError(1031 f"Appending path {path!r} starting from slash is forbidden"1032 )1033 # We need to quote the path if it is not already encoded1034 # This cannot be done at the end because the existing1035 # path is already quoted and we do not want to double quote1036 # the existing path.1037 path = path if encoded else PATH_QUOTER(path)1038 needs_normalize |= "." in path1039 segments = path.split("/")1040 segments.reverse()1041 # remove trailing empty segment for all but the last path1042 parsed += segments[1:] if not last and segments[0] == "" else segments1043 1044 if (path := self._path) and (old_segments := path.split("/")):1045 # If the old path ends with a slash, the last segment is an empty string1046 # and should be removed before adding the new path segments.1047 old = old_segments[:-1] if old_segments[-1] == "" else old_segments1048 old.reverse()1049 parsed += old1050 1051 # If the netloc is present, inject a leading slash when adding a1052 # path to an absolute URL where there was none before.1053 if (netloc := self._netloc) and parsed and parsed[-1] != "":1054 parsed.append("")1055 1056 parsed.reverse()1057 if not netloc or not needs_normalize:1058 return from_parts(self._scheme, netloc, "/".join(parsed), "", "")1059 1060 path = "/".join(normalize_path_segments(parsed))1061 # If normalizing the path segments removed the leading slash, add it back.1062 if path and path[0] != "/":1063 path = f"/{path}"1064 return from_parts(self._scheme, netloc, path, "", "")1065 1066 def with_scheme(self, scheme: str) -> "URL":1067 """Return a new URL with scheme replaced."""1068 # N.B. doesn't cleanup query/fragment1069 if not isinstance(scheme, str):1070 raise TypeError("Invalid scheme type")1071 lower_scheme = scheme.lower()1072 netloc = self._netloc1073 if not netloc and lower_scheme in SCHEME_REQUIRES_HOST:1074 msg = (1075 "scheme replacement is not allowed for "1076 f"relative URLs for the {lower_scheme} scheme"1077 )1078 raise ValueError(msg)1079 return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment)1080 1081 def with_user(self, user: str | None) -> "URL":1082 """Return a new URL with user replaced.1083 1084 Autoencode user if needed.1085 1086 Clear user/password if user is None.1087 1088 """1089 # N.B. doesn't cleanup query/fragment1090 if user is None:1091 password = None1092 elif isinstance(user, str):1093 user = QUOTER(user)1094 password = self.raw_password1095 else:1096 raise TypeError("Invalid user type")1097 if not (netloc := self._netloc):1098 raise ValueError("user replacement is not allowed for relative URLs")1099 encoded_host = self.host_subcomponent or ""1100 netloc = make_netloc(user, password, encoded_host, self.explicit_port)1101 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)1102 1103 def with_password(self, password: str | None) -> "URL":1104 """Return a new URL with password replaced.1105 1106 Autoencode password if needed.1107 1108 Clear password if argument is None.1109 1110 """1111 # N.B. doesn't cleanup query/fragment1112 if password is None:1113 pass1114 elif isinstance(password, str):1115 password = QUOTER(password)1116 else:1117 raise TypeError("Invalid password type")1118 if not (netloc := self._netloc):1119 raise ValueError("password replacement is not allowed for relative URLs")1120 encoded_host = self.host_subcomponent or ""1121 port = self.explicit_port1122 netloc = make_netloc(self.raw_user, password, encoded_host, port)1123 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)1124 1125 def with_host(self, host: str) -> "URL":1126 """Return a new URL with host replaced.1127 1128 Autoencode host if needed.1129 1130 Changing host for relative URLs is not allowed, use .join()1131 instead.1132 1133 """1134 # N.B. doesn't cleanup query/fragment1135 if not isinstance(host, str):1136 raise TypeError("Invalid host type")1137 if not (netloc := self._netloc):1138 raise ValueError("host replacement is not allowed for relative URLs")1139 if not host:1140 raise ValueError("host removing is not allowed")1141 encoded_host = _encode_host(host, validate_host=True) if host else ""1142 port = self.explicit_port1143 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)1144 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)1145 1146 def with_port(self, port: int | None) -> "URL":1147 """Return a new URL with port replaced.1148 1149 Clear port to default if None is passed.1150 1151 """1152 # N.B. doesn't cleanup query/fragment1153 if port is not None:1154 if isinstance(port, bool) or not isinstance(port, int):1155 raise TypeError(f"port should be int or None, got {type(port)}")1156 if not (0 <= port <= 65535):1157 raise ValueError(f"port must be between 0 and 65535, got {port}")1158 if not (netloc := self._netloc):1159 raise ValueError("port replacement is not allowed for relative URLs")1160 encoded_host = self.host_subcomponent or ""1161 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)1162 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)1163 1164 def with_path(1165 self,1166 path: str,1167 *,1168 encoded: bool = False,1169 keep_query: bool = False,1170 keep_fragment: bool = False,1171 ) -> "URL":1172 """Return a new URL with path replaced."""1173 netloc = self._netloc1174 if not encoded:1175 path = PATH_QUOTER(path)1176 if netloc:1177 path = normalize_path(path) if "." in path else path1178 if path and path[0] != "/":1179 path = f"/{path}"1180 query = self._query if keep_query else ""1181 fragment = self._fragment if keep_fragment else ""1182 return from_parts(self._scheme, netloc, path, query, fragment)1183 1184 @overload1185 def with_query(self, query: Query) -> "URL": ...1186 1187 @overload1188 def with_query(self, **kwargs: QueryVariable) -> "URL": ...1189 1190 def with_query(self, *args: Any, **kwargs: Any) -> "URL":1191 """Return a new URL with query part replaced.1192 1193 Accepts any Mapping (e.g. dict, multidict.MultiDict instances)1194 or str, autoencode the argument if needed.1195 1196 A sequence of (key, value) pairs is supported as well.1197 1198 It also can take an arbitrary number of keyword arguments.1199 1200 Clear query if None is passed.