codekingpro/portable-devtools

1# Copyright Joyent, Inc. and other Node contributors.2#3# Permission is hereby granted, free of charge, to any person obtaining a4# copy of this software and associated documentation files (the5# "Software"), to deal in the Software without restriction, including6# without limitation the rights to use, copy, modify, merge, publish,7# distribute, sublicense, and/or sell copies of the Software, and to permit8# persons to whom the Software is furnished to do so, subject to the9# following conditions:10#11# The above copyright notice and this permission notice shall be included12# in all copies or substantial portions of the Software.13#14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF16# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN17# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,18# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR19# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE20# USE OR OTHER DEALINGS IN THE SOFTWARE.21 22 23# Changes from joyent/node:24#25# 1. No leading slash in paths,26#    e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/`27#28# 2. Backslashes are not replaced with slashes,29#    so `http:\\example.org\` is treated like a relative path30#31# 3. Trailing colon is treated like a part of the path,32#    i.e. in `http://example.org:foo` pathname is `:foo`33#34# 4. Nothing is URL-encoded in the resulting object,35#    (in joyent/node some chars in auth and paths are encoded)36#37# 5. `url.parse()` does not have `parseQueryString` argument38#39# 6. Removed extraneous result properties: `host`, `path`, `query`, etc.,40#    which can be constructed using other parts of the url.41 42from __future__ import annotations43 44from collections import defaultdict45import re46 47from mdurl._url import URL48 49# Reference: RFC 3986, RFC 1808, RFC 239650 51# define these here so at least they only have to be52# compiled once on the first module load.53PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE)54PORT_PATTERN = re.compile(r":[0-9]*$")55 56# Special case for a simple path URL57SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$")58 59# RFC 2396: characters reserved for delimiting URLs.60# We actually just auto-escape these.61DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t")62 63# RFC 2396: characters not allowed for various reasons.64UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS65 66# Allowed by RFCs, but cause of XSS attacks.  Always escape these.67AUTO_ESCAPE = ("'",) + UNWISE68# Characters that are never ever allowed in a hostname.69# Note that any invalid chars are also handled, but these70# are the ones that are *expected* to be seen, so we fast-path71# them.72NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE73HOST_ENDING_CHARS = ("/", "?", "#")74HOSTNAME_MAX_LEN = 25575HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$")76HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$")77# protocols that can allow "unsafe" and "unwise" chars.78 79# protocols that never have a hostname.80HOSTLESS_PROTOCOL = defaultdict(81    bool,82    {83        "javascript": True,84        "javascript:": True,85    },86)87# protocols that always contain a // bit.88SLASHED_PROTOCOL = defaultdict(89    bool,90    {91        "http": True,92        "https": True,93        "ftp": True,94        "gopher": True,95        "file": True,96        "http:": True,97        "https:": True,98        "ftp:": True,99        "gopher:": True,100        "file:": True,101    },102)103 104 105class MutableURL:106    def __init__(self) -> None:107        self.protocol: str | None = None108        self.slashes: bool = False109        self.auth: str | None = None110        self.port: str | None = None111        self.hostname: str | None = None112        self.hash: str | None = None113        self.search: str | None = None114        self.pathname: str | None = None115 116    def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL":117        lower_proto = ""118        slashes = False119        rest = url120 121        # trim before proceeding.122        # This is to support parse stuff like "  http://foo.com  \n"123        rest = rest.strip()124 125        if not slashes_denote_host and len(url.split("#")) == 1:126            # Try fast path regexp127            simple_path = SIMPLE_PATH_PATTERN.match(rest)128            if simple_path:129                self.pathname = simple_path.group(1)130                if simple_path.group(2):131                    self.search = simple_path.group(2)132                return self133 134        proto = ""135        proto_match = PROTOCOL_PATTERN.match(rest)136        if proto_match:137            proto = proto_match.group()138            lower_proto = proto.lower()139            self.protocol = proto140            rest = rest[len(proto) :]141 142        # figure out if it's got a host143        # user@server is *always* interpreted as a hostname, and url144        # resolution will treat //foo/bar as host=foo,path=bar because that's145        # how the browser resolves relative URLs.146        if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest):147            slashes = rest.startswith("//")148            if slashes and not (proto and HOSTLESS_PROTOCOL[proto]):149                rest = rest[2:]150                self.slashes = True151 152        if not HOSTLESS_PROTOCOL[proto] and (153            slashes or (proto and not SLASHED_PROTOCOL[proto])154        ):155 156            # there's a hostname.157            # the first instance of /, ?, ;, or # ends the host.158            #159            # If there is an @ in the hostname, then non-host chars *are* allowed160            # to the left of the last @ sign, unless some host-ending character161            # comes *before* the @-sign.162            # URLs are obnoxious.163            #164            # ex:165            # http://a@b@c/ => user:a@b host:c166            # http://a@b?@c => user:a host:c path:/?@c167 168            # v0.12 TODO(isaacs): This is not quite how Chrome does things.169            # Review our test case against browsers more comprehensively.170 171            # find the first instance of any hostEndingChars172            host_end = -1173            for i in range(len(HOST_ENDING_CHARS)):174                hec = rest.find(HOST_ENDING_CHARS[i])175                if hec != -1 and (host_end == -1 or hec < host_end):176                    host_end = hec177 178            # at this point, either we have an explicit point where the179            # auth portion cannot go past, or the last @ char is the decider.180            if host_end == -1:181                # atSign can be anywhere.182                at_sign = rest.rfind("@")183            else:184                # atSign must be in auth portion.185                # http://a@b/c@d => host:b auth:a path:/c@d186                at_sign = rest.rfind("@", 0, host_end + 1)187 188            # Now we have a portion which is definitely the auth.189            # Pull that off.190            if at_sign != -1:191                auth = rest[:at_sign]192                rest = rest[at_sign + 1 :]193                self.auth = auth194 195            # the host is the remaining to the left of the first non-host char196            host_end = -1197            for i in range(len(NON_HOST_CHARS)):198                hec = rest.find(NON_HOST_CHARS[i])199                if hec != -1 and (host_end == -1 or hec < host_end):200                    host_end = hec201            # if we still have not hit it, then the entire thing is a host.202            if host_end == -1:203                host_end = len(rest)204 205            if host_end > 0 and rest[host_end - 1] == ":":206                host_end -= 1207            host = rest[:host_end]208            rest = rest[host_end:]209 210            # pull out port.211            self.parse_host(host)212 213            # we've indicated that there is a hostname,214            # so even if it's empty, it has to be present.215            self.hostname = self.hostname or ""216 217            # if hostname begins with [ and ends with ]218            # assume that it's an IPv6 address.219            ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith(220                "]"221            )222 223            # validate a little.224            if not ipv6_hostname:225                hostparts = self.hostname.split(".")226                l = len(hostparts)  # noqa: E741227                i = 0228                while i < l:229                    part = hostparts[i]230                    if not part:231                        i += 1  # emulate statement3 in JS for loop232                        continue233                    if not HOSTNAME_PART_PATTERN.search(part):234                        newpart = ""235                        k = len(part)236                        j = 0237                        while j < k:238                            if ord(part[j]) > 127:239                                # we replace non-ASCII char with a temporary placeholder240                                # we need this to make sure size of hostname is not241                                # broken by replacing non-ASCII by nothing242                                newpart += "x"243                            else:244                                newpart += part[j]245                            j += 1  # emulate statement3 in JS for loop246 247                        # we test again with ASCII char only248                        if not HOSTNAME_PART_PATTERN.search(newpart):249                            valid_parts = hostparts[:i]250                            not_host = hostparts[i + 1 :]251                            bit = HOSTNAME_PART_START.search(part)252                            if bit:253                                valid_parts.append(bit.group(1))254                                not_host.insert(0, bit.group(2))255                            if not_host:256                                rest = ".".join(not_host) + rest257                            self.hostname = ".".join(valid_parts)258                            break259                    i += 1  # emulate statement3 in JS for loop260 261            if len(self.hostname) > HOSTNAME_MAX_LEN:262                self.hostname = ""263 264            # strip [ and ] from the hostname265            # the host field still retains them, though266            if ipv6_hostname:267                self.hostname = self.hostname[1:-1]268 269        # chop off from the tail first.270        hash = rest.find("#")  # noqa: A001271        if hash != -1:272            # got a fragment string.273            self.hash = rest[hash:]274            rest = rest[:hash]275        qm = rest.find("?")276        if qm != -1:277            self.search = rest[qm:]278            rest = rest[:qm]279        if rest:280            self.pathname = rest281        if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname:282            self.pathname = ""283 284        return self285 286    def parse_host(self, host: str) -> None:287        port_match = PORT_PATTERN.search(host)288        if port_match:289            port = port_match.group()290            if port != ":":291                self.port = port[1:]292            host = host[: -len(port)]293        if host:294            self.hostname = host295 296 297def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL:298    if isinstance(url, URL):299        return url300    u = MutableURL()301    u.parse(url, slashes_denote_host)302    return URL(303        u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname304    )305