codekingpro/portable-devtools

1import re2import time3import typing4from collections.abc import Iterable5 6from mitmproxy.http import Headers7from mitmproxy.http import Request8from mitmproxy.http import Response9from mitmproxy.net.http import url10from mitmproxy.net.http import validate11 12 13def get_header_tokens(headers, key):14    """15    Retrieve all tokens for a header key. A number of different headers16    follow a pattern where each header line can containe comma-separated17    tokens, and headers can be set multiple times.18    """19    if key not in headers:20        return []21    tokens = headers[key].split(",")22    return [token.strip() for token in tokens]23 24 25def connection_close(http_version, headers):26    """27    Checks the message to see if the client connection should be closed28    according to RFC 2616 Section 8.1.29    If we don't have a Connection header, HTTP 1.1 connections are assumed30    to be persistent.31    """32    if "connection" in headers:33        tokens = get_header_tokens(headers, "connection")34        if "close" in tokens:35            return True36        elif "keep-alive" in tokens:37            return False38 39    return http_version not in (40        "HTTP/1.1",41        b"HTTP/1.1",42        "HTTP/2.0",43        b"HTTP/2.0",44    )45 46 47def expected_http_body_size(48    request: Request, response: Response | None = None49) -> int | None:50    """51    Returns:52        The expected body length:53        - a positive integer, if the size is known in advance54        - None, if the size in unknown in advance (chunked encoding)55        - -1, if all data should be read until end of stream.56 57    Raises:58        ValueError, if the content-length or transfer-encoding header is invalid59    """60    # Determine response size according to http://tools.ietf.org/html/rfc7230#section-3.3, which is inlined below.61    if not response:62        headers = request.headers63    else:64        headers = response.headers65 66        #    1.  Any response to a HEAD request and any response with a 1xx67        #        (Informational), 204 (No Content), or 304 (Not Modified) status68        #        code is always terminated by the first empty line after the69        #        header fields, regardless of the header fields present in the70        #        message, and thus cannot contain a message body.71        if request.method.upper() == "HEAD":72            return 073        if 100 <= response.status_code <= 199:74            return 075        if response.status_code in (204, 304):76            return 077 78        #    2.  Any 2xx (Successful) response to a CONNECT request implies that79        #        the connection will become a tunnel immediately after the empty80        #        line that concludes the header fields.  A client MUST ignore any81        #        Content-Length or Transfer-Encoding header fields received in82        #        such a message.83        if 200 <= response.status_code <= 299 and request.method.upper() == "CONNECT":84            return 085 86    #    3.  If a Transfer-Encoding header field is present and the chunked87    #        transfer coding (Section 4.1) is the final encoding, the message88    #        body length is determined by reading and decoding the chunked89    #        data until the transfer coding indicates the data is complete.90    #91    #        If a Transfer-Encoding header field is present in a response and92    #        the chunked transfer coding is not the final encoding, the93    #        message body length is determined by reading the connection until94    #        it is closed by the server.  If a Transfer-Encoding header field95    #        is present in a request and the chunked transfer coding is not96    #        the final encoding, the message body length cannot be determined97    #        reliably; the server MUST respond with the 400 (Bad Request)98    #        status code and then close the connection.99    #100    #        If a message is received with both a Transfer-Encoding and a101    #        Content-Length header field, the Transfer-Encoding overrides the102    #        Content-Length.  Such a message might indicate an attempt to103    #        perform request smuggling (Section 9.5) or response splitting104    #        (Section 9.4) and ought to be handled as an error.  A sender MUST105    #        remove the received Content-Length field prior to forwarding such106    #        a message downstream.107    #108    if te_str := headers.get("transfer-encoding"):109        te = validate.parse_transfer_encoding(te_str)110        match te:111            case "chunked" | "compress,chunked" | "deflate,chunked" | "gzip,chunked":112                return None113            case "compress" | "deflate" | "gzip" | "identity":114                if response:115                    return -1116                # These values are valid for responses only (not requests), which is ensured in117                # mitmproxy.net.http.validate. If users have explicitly disabled header validation,118                # we strive for maximum compatibility with weird clients.119                if te == "identity" or "content-length" in headers:120                    pass  # Content-Length or 0121                else:122                    return (123                        -1124                    )  # compress/deflate/gzip with no content-length -> read until eof125            case other:  # pragma: no cover126                typing.assert_never(other)127 128    #    4.  If a message is received without Transfer-Encoding and with129    #        either multiple Content-Length header fields having differing130    #        field-values or a single Content-Length header field having an131    #        invalid value, then the message framing is invalid and the132    #        recipient MUST treat it as an unrecoverable error.  If this is a133    #        request message, the server MUST respond with a 400 (Bad Request)134    #        status code and then close the connection.  If this is a response135    #        message received by a proxy, the proxy MUST close the connection136    #        to the server, discard the received response, and send a 502 (Bad137    #        Gateway) response to the client.  If this is a response message138    #        received by a user agent, the user agent MUST close the139    #        connection to the server and discard the received response.140    #141    #    5.  If a valid Content-Length header field is present without142    #        Transfer-Encoding, its decimal value defines the expected message143    #        body length in octets.  If the sender closes the connection or144    #        the recipient times out before the indicated number of octets are145    #        received, the recipient MUST consider the message to be146    #        incomplete and close the connection.147    if cl := headers.get("content-length"):148        return validate.parse_content_length(cl)149    #    6.  If this is a request message and none of the above are true, then150    #        the message body length is zero (no message body is present).151    if not response:152        return 0153 154    #    7.  Otherwise, this is a response message without a declared message155    #        body length, so the message body length is determined by the156    #        number of octets received prior to the server closing the157    #        connection.158    return -1159 160 161def raise_if_http_version_unknown(http_version: bytes) -> None:162    if not re.match(rb"^HTTP/\d\.\d$", http_version):163        raise ValueError(f"Unknown HTTP version: {http_version!r}")164 165 166def _read_request_line(167    line: bytes,168) -> tuple[str, int, bytes, bytes, bytes, bytes, bytes]:169    try:170        method, target, http_version = line.split()171        port: int | None172 173        if target == b"*" or target.startswith(b"/"):174            scheme, authority, path = b"", b"", target175            host, port = "", 0176        elif method == b"CONNECT":177            scheme, authority, path = b"", target, b""178            host, port = url.parse_authority(authority, check=True)179            if not port:180                raise ValueError181        else:182            scheme, rest = target.split(b"://", maxsplit=1)183            # https://www.rfc-editor.org/rfc/rfc3986.html#section-3.1184            # An implementation should accept uppercase letters as equivalent to lowercase in scheme names185            # (e.g., allow "HTTP" as well as "http") for the sake of robustness but should only produce186            # lowercase scheme names for consistency.187            scheme = scheme.lower()188            authority, _, path_ = rest.partition(b"/")189            path = b"/" + path_190            host, port = url.parse_authority(authority, check=True)191            port = port or url.default_port(scheme)192            if not port:193                raise ValueError194            # TODO: we can probably get rid of this check?195            url.parse(target)196 197        raise_if_http_version_unknown(http_version)198    except ValueError as e:199        raise ValueError(f"Bad HTTP request line: {line!r}") from e200 201    return host, port, method, scheme, authority, path, http_version202 203 204def _read_response_line(line: bytes) -> tuple[bytes, int, bytes]:205    try:206        parts = line.split(None, 2)207        if len(parts) == 2:  # handle missing message gracefully208            parts.append(b"")209 210        http_version, status_code_str, reason = parts211        status_code = int(status_code_str)212        raise_if_http_version_unknown(http_version)213    except ValueError as e:214        raise ValueError(f"Bad HTTP response line: {line!r}") from e215 216    return http_version, status_code, reason217 218 219def _read_headers(lines: Iterable[bytes]) -> Headers:220    """221    Read a set of headers.222    Stop once a blank line is reached.223 224    Returns:225        A headers object226 227    Raises:228        exceptions.HttpSyntaxException229    """230    ret: list[tuple[bytes, bytes]] = []231    for line in lines:232        if line[0] in b" \t":233            if not ret:234                raise ValueError("Invalid headers")235            # continued header236            ret[-1] = (ret[-1][0], ret[-1][1] + b"\r\n " + line.strip())237        else:238            try:239                name, value = line.split(b":", 1)240                value = value.strip()241                if not name:242                    raise ValueError()243                ret.append((name, value))244            except ValueError:245                raise ValueError(f"Invalid header line: {line!r}")246    return Headers(ret)247 248 249def read_request_head(lines: list[bytes]) -> Request:250    """251    Parse an HTTP request head (request line + headers) from an iterable of lines252 253    Args:254        lines: The input lines255 256    Returns:257        The HTTP request object (without body)258 259    Raises:260        ValueError: The input is malformed.261    """262    host, port, method, scheme, authority, path, http_version = _read_request_line(263        lines[0]264    )265    headers = _read_headers(lines[1:])266 267    return Request(268        host=host,269        port=port,270        method=method,271        scheme=scheme,272        authority=authority,273        path=path,274        http_version=http_version,275        headers=headers,276        content=None,277        trailers=None,278        timestamp_start=time.time(),279        timestamp_end=None,280    )281 282 283def read_response_head(lines: list[bytes]) -> Response:284    """285    Parse an HTTP response head (response line + headers) from an iterable of lines286 287    Args:288        lines: The input lines289 290    Returns:291        The HTTP response object (without body)292 293    Raises:294        ValueError: The input is malformed.295    """296    http_version, status_code, reason = _read_response_line(lines[0])297    headers = _read_headers(lines[1:])298 299    return Response(300        http_version=http_version,301        status_code=status_code,302        reason=reason,303        headers=headers,304        content=None,305        trailers=None,306        timestamp_start=time.time(),307        timestamp_end=None,308    )309