codekingpro/portable-devtools
115k
1"""2An implementation of `urlparse` that provides URL validation and normalization3as described by RFC3986.4 5We rely on this implementation rather than the one in Python's stdlib, because:6 7* It provides more complete URL validation.8* It properly differentiates between an empty querystring and an absent querystring,9 to distinguish URLs with a trailing '?'.10* It handles scheme, hostname, port, and path normalization.11* It supports IDNA hostnames, normalizing them to their encoded form.12* The API supports passing individual components, as well as the complete URL string.13 14Previously we relied on the excellent `rfc3986` package to handle URL parsing and15validation, but this module provides a simpler alternative, with less indirection16required.17"""18 19from __future__ import annotations20 21import ipaddress22import re23import typing24 25import idna26 27from ._exceptions import InvalidURL28 29MAX_URL_LENGTH = 6553630 31# https://datatracker.ietf.org/doc/html/rfc3986.html#section-2.332UNRESERVED_CHARACTERS = (33 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"34)35SUB_DELIMS = "!$&'()*+,;="36 37PERCENT_ENCODED_REGEX = re.compile("%[A-Fa-f0-9]{2}")38 39# https://url.spec.whatwg.org/#percent-encoded-bytes40 41# The fragment percent-encode set is the C0 control percent-encode set42# and U+0020 SPACE, U+0022 ("), U+003C (<), U+003E (>), and U+0060 (`).43FRAG_SAFE = "".join(44 [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x3C, 0x3E, 0x60)]45)46 47# The query percent-encode set is the C0 control percent-encode set48# and U+0020 SPACE, U+0022 ("), U+0023 (#), U+003C (<), and U+003E (>).49QUERY_SAFE = "".join(50 [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E)]51)52 53# The path percent-encode set is the query percent-encode set54# and U+003F (?), U+0060 (`), U+007B ({), and U+007D (}).55PATH_SAFE = "".join(56 [57 chr(i)58 for i in range(0x20, 0x7F)59 if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + (0x3F, 0x60, 0x7B, 0x7D)60 ]61)62 63# The userinfo percent-encode set is the path percent-encode set64# and U+002F (/), U+003A (:), U+003B (;), U+003D (=), U+0040 (@),65# U+005B ([) to U+005E (^), inclusive, and U+007C (|).66USERNAME_SAFE = "".join(67 [68 chr(i)69 for i in range(0x20, 0x7F)70 if i71 not in (0x20, 0x22, 0x23, 0x3C, 0x3E)72 + (0x3F, 0x60, 0x7B, 0x7D)73 + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C)74 ]75)76PASSWORD_SAFE = "".join(77 [78 chr(i)79 for i in range(0x20, 0x7F)80 if i81 not in (0x20, 0x22, 0x23, 0x3C, 0x3E)82 + (0x3F, 0x60, 0x7B, 0x7D)83 + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C)84 ]85)86# Note... The terminology 'userinfo' percent-encode set in the WHATWG document87# is used for the username and password quoting. For the joint userinfo component88# we remove U+003A (:) from the safe set.89USERINFO_SAFE = "".join(90 [91 chr(i)92 for i in range(0x20, 0x7F)93 if i94 not in (0x20, 0x22, 0x23, 0x3C, 0x3E)95 + (0x3F, 0x60, 0x7B, 0x7D)96 + (0x2F, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C)97 ]98)99 100 101# {scheme}: (optional)102# //{authority} (optional)103# {path}104# ?{query} (optional)105# #{fragment} (optional)106URL_REGEX = re.compile(107 (108 r"(?:(?P<scheme>{scheme}):)?"109 r"(?://(?P<authority>{authority}))?"110 r"(?P<path>{path})"111 r"(?:\?(?P<query>{query}))?"112 r"(?:#(?P<fragment>{fragment}))?"113 ).format(114 scheme="([a-zA-Z][a-zA-Z0-9+.-]*)?",115 authority="[^/?#]*",116 path="[^?#]*",117 query="[^#]*",118 fragment=".*",119 )120)121 122# {userinfo}@ (optional)123# {host}124# :{port} (optional)125AUTHORITY_REGEX = re.compile(126 (127 r"(?:(?P<userinfo>{userinfo})@)?" r"(?P<host>{host})" r":?(?P<port>{port})?"128 ).format(129 userinfo=".*", # Any character sequence.130 host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@',131 # or an IPv6 address enclosed within square brackets.132 port=".*", # Any character sequence.133 )134)135 136 137# If we call urlparse with an individual component, then we need to regex138# validate that component individually.139# Note that we're duplicating the same strings as above. Shock! Horror!!140COMPONENT_REGEX = {141 "scheme": re.compile("([a-zA-Z][a-zA-Z0-9+.-]*)?"),142 "authority": re.compile("[^/?#]*"),143 "path": re.compile("[^?#]*"),144 "query": re.compile("[^#]*"),145 "fragment": re.compile(".*"),146 "userinfo": re.compile("[^@]*"),147 "host": re.compile("(\\[.*\\]|[^:]*)"),148 "port": re.compile(".*"),149}150 151 152# We use these simple regexs as a first pass before handing off to153# the stdlib 'ipaddress' module for IP address validation.154IPv4_STYLE_HOSTNAME = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$")155IPv6_STYLE_HOSTNAME = re.compile(r"^\[.*\]$")156 157 158class ParseResult(typing.NamedTuple):159 scheme: str160 userinfo: str161 host: str162 port: int | None163 path: str164 query: str | None165 fragment: str | None166 167 @property168 def authority(self) -> str:169 return "".join(170 [171 f"{self.userinfo}@" if self.userinfo else "",172 f"[{self.host}]" if ":" in self.host else self.host,173 f":{self.port}" if self.port is not None else "",174 ]175 )176 177 @property178 def netloc(self) -> str:179 return "".join(180 [181 f"[{self.host}]" if ":" in self.host else self.host,182 f":{self.port}" if self.port is not None else "",183 ]184 )185 186 def copy_with(self, **kwargs: str | None) -> ParseResult:187 if not kwargs:188 return self189 190 defaults = {191 "scheme": self.scheme,192 "authority": self.authority,193 "path": self.path,194 "query": self.query,195 "fragment": self.fragment,196 }197 defaults.update(kwargs)198 return urlparse("", **defaults)199 200 def __str__(self) -> str:201 authority = self.authority202 return "".join(203 [204 f"{self.scheme}:" if self.scheme else "",205 f"//{authority}" if authority else "",206 self.path,207 f"?{self.query}" if self.query is not None else "",208 f"#{self.fragment}" if self.fragment is not None else "",209 ]210 )211 212 213def urlparse(url: str = "", **kwargs: str | None) -> ParseResult:214 # Initial basic checks on allowable URLs.215 # ---------------------------------------216 217 # Hard limit the maximum allowable URL length.218 if len(url) > MAX_URL_LENGTH:219 raise InvalidURL("URL too long")220 221 # If a URL includes any ASCII control characters including \t, \r, \n,222 # then treat it as invalid.223 if any(char.isascii() and not char.isprintable() for char in url):224 char = next(char for char in url if char.isascii() and not char.isprintable())225 idx = url.find(char)226 error = (227 f"Invalid non-printable ASCII character in URL, {char!r} at position {idx}."228 )229 raise InvalidURL(error)230 231 # Some keyword arguments require special handling.232 # ------------------------------------------------233 234 # Coerce "port" to a string, if it is provided as an integer.235 if "port" in kwargs:236 port = kwargs["port"]237 kwargs["port"] = str(port) if isinstance(port, int) else port238 239 # Replace "netloc" with "host and "port".240 if "netloc" in kwargs:241 netloc = kwargs.pop("netloc") or ""242 kwargs["host"], _, kwargs["port"] = netloc.partition(":")243 244 # Replace "username" and/or "password" with "userinfo".245 if "username" in kwargs or "password" in kwargs:246 username = quote(kwargs.pop("username", "") or "", safe=USERNAME_SAFE)247 password = quote(kwargs.pop("password", "") or "", safe=PASSWORD_SAFE)248 kwargs["userinfo"] = f"{username}:{password}" if password else username249 250 # Replace "raw_path" with "path" and "query".251 if "raw_path" in kwargs:252 raw_path = kwargs.pop("raw_path") or ""253 kwargs["path"], seperator, kwargs["query"] = raw_path.partition("?")254 if not seperator:255 kwargs["query"] = None256 257 # Ensure that IPv6 "host" addresses are always escaped with "[...]".258 if "host" in kwargs:259 host = kwargs.get("host") or ""260 if ":" in host and not (host.startswith("[") and host.endswith("]")):261 kwargs["host"] = f"[{host}]"262 263 # If any keyword arguments are provided, ensure they are valid.264 # -------------------------------------------------------------265 266 for key, value in kwargs.items():267 if value is not None:268 if len(value) > MAX_URL_LENGTH:269 raise InvalidURL(f"URL component '{key}' too long")270 271 # If a component includes any ASCII control characters including \t, \r, \n,272 # then treat it as invalid.273 if any(char.isascii() and not char.isprintable() for char in value):274 char = next(275 char for char in value if char.isascii() and not char.isprintable()276 )277 idx = value.find(char)278 error = (279 f"Invalid non-printable ASCII character in URL {key} component, "280 f"{char!r} at position {idx}."281 )282 raise InvalidURL(error)283 284 # Ensure that keyword arguments match as a valid regex.285 if not COMPONENT_REGEX[key].fullmatch(value):286 raise InvalidURL(f"Invalid URL component '{key}'")287 288 # The URL_REGEX will always match, but may have empty components.289 url_match = URL_REGEX.match(url)290 assert url_match is not None291 url_dict = url_match.groupdict()292 293 # * 'scheme', 'authority', and 'path' may be empty strings.294 # * 'query' may be 'None', indicating no trailing "?" portion.295 # Any string including the empty string, indicates a trailing "?".296 # * 'fragment' may be 'None', indicating no trailing "#" portion.297 # Any string including the empty string, indicates a trailing "#".298 scheme = kwargs.get("scheme", url_dict["scheme"]) or ""299 authority = kwargs.get("authority", url_dict["authority"]) or ""300 path = kwargs.get("path", url_dict["path"]) or ""301 query = kwargs.get("query", url_dict["query"])302 frag = kwargs.get("fragment", url_dict["fragment"])303 304 # The AUTHORITY_REGEX will always match, but may have empty components.305 authority_match = AUTHORITY_REGEX.match(authority)306 assert authority_match is not None307 authority_dict = authority_match.groupdict()308 309 # * 'userinfo' and 'host' may be empty strings.310 # * 'port' may be 'None'.311 userinfo = kwargs.get("userinfo", authority_dict["userinfo"]) or ""312 host = kwargs.get("host", authority_dict["host"]) or ""313 port = kwargs.get("port", authority_dict["port"])314 315 # Normalize and validate each component.316 # We end up with a parsed representation of the URL,317 # with components that are plain ASCII bytestrings.318 parsed_scheme: str = scheme.lower()319 parsed_userinfo: str = quote(userinfo, safe=USERINFO_SAFE)320 parsed_host: str = encode_host(host)321 parsed_port: int | None = normalize_port(port, scheme)322 323 has_scheme = parsed_scheme != ""324 has_authority = (325 parsed_userinfo != "" or parsed_host != "" or parsed_port is not None326 )327 validate_path(path, has_scheme=has_scheme, has_authority=has_authority)328 if has_scheme or has_authority:329 path = normalize_path(path)330 331 parsed_path: str = quote(path, safe=PATH_SAFE)332 parsed_query: str | None = None if query is None else quote(query, safe=QUERY_SAFE)333 parsed_frag: str | None = None if frag is None else quote(frag, safe=FRAG_SAFE)334 335 # The parsed ASCII bytestrings are our canonical form.336 # All properties of the URL are derived from these.337 return ParseResult(338 parsed_scheme,339 parsed_userinfo,340 parsed_host,341 parsed_port,342 parsed_path,343 parsed_query,344 parsed_frag,345 )346 347 348def encode_host(host: str) -> str:349 if not host:350 return ""351 352 elif IPv4_STYLE_HOSTNAME.match(host):353 # Validate IPv4 hostnames like #.#.#.#354 #355 # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2356 #357 # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet358 try:359 ipaddress.IPv4Address(host)360 except ipaddress.AddressValueError:361 raise InvalidURL(f"Invalid IPv4 address: {host!r}")362 return host363 364 elif IPv6_STYLE_HOSTNAME.match(host):365 # Validate IPv6 hostnames like [...]366 #367 # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2368 #369 # "A host identified by an Internet Protocol literal address, version 6370 # [RFC3513] or later, is distinguished by enclosing the IP literal371 # within square brackets ("[" and "]"). This is the only place where372 # square bracket characters are allowed in the URI syntax."373 try:374 ipaddress.IPv6Address(host[1:-1])375 except ipaddress.AddressValueError:376 raise InvalidURL(f"Invalid IPv6 address: {host!r}")377 return host[1:-1]378 379 elif host.isascii():380 # Regular ASCII hostnames381 #382 # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2383 #384 # reg-name = *( unreserved / pct-encoded / sub-delims )385 WHATWG_SAFE = '"`{}%|\\'386 return quote(host.lower(), safe=SUB_DELIMS + WHATWG_SAFE)387 388 # IDNA hostnames389 try:390 return idna.encode(host.lower()).decode("ascii")391 except idna.IDNAError:392 raise InvalidURL(f"Invalid IDNA hostname: {host!r}")393 394 395def normalize_port(port: str | int | None, scheme: str) -> int | None:396 # From https://tools.ietf.org/html/rfc3986#section-3.2.3397 #398 # "A scheme may define a default port. For example, the "http" scheme399 # defines a default port of "80", corresponding to its reserved TCP400 # port number. The type of port designated by the port number (e.g.,401 # TCP, UDP, SCTP) is defined by the URI scheme. URI producers and402 # normalizers should omit the port component and its ":" delimiter if403 # port is empty or if its value would be the same as that of the404 # scheme's default."405 if port is None or port == "":406 return None407 408 try:409 port_as_int = int(port)410 except ValueError:411 raise InvalidURL(f"Invalid port: {port!r}")412 413 # See https://url.spec.whatwg.org/#url-miscellaneous414 default_port = {"ftp": 21, "http": 80, "https": 443, "ws": 80, "wss": 443}.get(415 scheme416 )417 if port_as_int == default_port:418 return None419 return port_as_int420 421 422def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None:423 """424 Path validation rules that depend on if the URL contains425 a scheme or authority component.426 427 See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3428 """429 if has_authority:430 # If a URI contains an authority component, then the path component431 # must either be empty or begin with a slash ("/") character."432 if path and not path.startswith("/"):433 raise InvalidURL("For absolute URLs, path must be empty or begin with '/'")434 435 if not has_scheme and not has_authority:436 # If a URI does not contain an authority component, then the path cannot begin437 # with two slash characters ("//").438 if path.startswith("//"):439 raise InvalidURL("Relative URLs cannot have a path starting with '//'")440 441 # In addition, a URI reference (Section 4.1) may be a relative-path reference,442 # in which case the first path segment cannot contain a colon (":") character.443 if path.startswith(":"):444 raise InvalidURL("Relative URLs cannot have a path starting with ':'")445 446 447def normalize_path(path: str) -> str:448 """449 Drop "." and ".." segments from a URL path.450 451 For example:452 453 normalize_path("/path/./to/somewhere/..") == "/path/to"454 """455 # Fast return when no '.' characters in the path.456 if "." not in path:457 return path458 459 components = path.split("/")460 461 # Fast return when no '.' or '..' components in the path.462 if "." not in components and ".." not in components:463 return path464 465 # https://datatracker.ietf.org/doc/html/rfc3986#section-5.2.4466 output: list[str] = []467 for component in components:468 if component == ".":469 pass470 elif component == "..":471 if output and output != [""]:472 output.pop()473 else:474 output.append(component)475 return "/".join(output)476 477 478def PERCENT(string: str) -> str:479 return "".join([f"%{byte:02X}" for byte in string.encode("utf-8")])480 481 482def percent_encoded(string: str, safe: str) -> str:483 """484 Use percent-encoding to quote a string.485 """486 NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe487 488 # Fast path for strings that don't need escaping.489 if not string.rstrip(NON_ESCAPED_CHARS):490 return string491 492 return "".join(493 [char if char in NON_ESCAPED_CHARS else PERCENT(char) for char in string]494 )495 496 497def quote(string: str, safe: str) -> str:498 """499 Use percent-encoding to quote a string, omitting existing '%xx' escape sequences.500 501 See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1502 503 * `string`: The string to be percent-escaped.504 * `safe`: A string containing characters that may be treated as safe, and do not505 need to be escaped. Unreserved characters are always treated as safe.506 See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3507 """508 parts = []509 current_position = 0510 for match in re.finditer(PERCENT_ENCODED_REGEX, string):511 start_position, end_position = match.start(), match.end()512 matched_text = match.group(0)513 # Add any text up to the '%xx' escape sequence.514 if start_position != current_position:515 leading_text = string[current_position:start_position]516 parts.append(percent_encoded(leading_text, safe=safe))517 518 # Add the '%xx' escape sequence.519 parts.append(matched_text)520 current_position = end_position521 522 # Add any text after the final '%xx' escape sequence.523 if current_position != len(string):524 trailing_text = string[current_position:]525 parts.append(percent_encoded(trailing_text, safe=safe))526 527 return "".join(parts)528