codekingpro/portable-devtools

1import logging2import re3import typing4 5from mitmproxy.http import Message6from mitmproxy.http import Request7from mitmproxy.http import Response8 9logger = logging.getLogger(__name__)10 11# https://datatracker.ietf.org/doc/html/rfc7230#section-3.2: Header fields are tokens.12# "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /  "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA13_valid_header_name = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-zA-Z]+$")14 15_valid_content_length = re.compile(rb"^(?:0|[1-9][0-9]*)$")16_valid_content_length_str = re.compile(r"^(?:0|[1-9][0-9]*)$")17 18# https://datatracker.ietf.org/doc/html/rfc9112#section-6.1:19# > A sender MUST NOT apply the chunked transfer coding more than once to a message body (i.e., chunking an already20# > chunked message is not allowed). If any transfer coding other than chunked is applied to a request's content, the21# > sender MUST apply chunked as the final transfer coding to ensure that the message is properly framed. If any22# > transfer coding other than chunked is applied to a response's content, the sender MUST either apply chunked as the23# > final transfer coding or terminate the message by closing the connection.24#25# The RFC technically still allows for fun encodings, we are a bit stricter and only accept a known subset by default.26TransferEncoding = typing.Literal[27    "chunked",28    "compress,chunked",29    "deflate,chunked",30    "gzip,chunked",31    "compress",32    "deflate",33    "gzip",34    "identity",35]36_HTTP_1_1_TRANSFER_ENCODINGS = frozenset(typing.get_args(TransferEncoding))37 38 39def parse_content_length(value: str | bytes) -> int:40    """Parse a content-length header value, or raise a ValueError if it is invalid."""41    if isinstance(value, str):42        valid = bool(_valid_content_length_str.match(value))43    else:44        valid = bool(_valid_content_length.match(value))45    if not valid:46        raise ValueError(f"invalid content-length header: {value!r}")47    return int(value)48 49 50def parse_transfer_encoding(value: str | bytes) -> TransferEncoding:51    """Parse a transfer-encoding header value, or raise a ValueError if it is invalid or unknown."""52    # guard against .lower() transforming non-ascii to ascii53    if not value.isascii():54        raise ValueError(f"invalid transfer-encoding header: {value!r}")55    if isinstance(value, str):56        te = value57    else:58        te = value.decode()59    te = te.lower()60    te = re.sub(r"[\t ]*,[\t ]*", ",", te)61    if te not in _HTTP_1_1_TRANSFER_ENCODINGS:62        raise ValueError(f"unknown transfer-encoding header: {value!r}")63    return typing.cast(TransferEncoding, te)64 65 66def validate_headers(message: Message) -> None:67    """68    Validate HTTP message headers to avoid request smuggling attacks.69 70    Raises a ValueError if they are malformed.71    """72 73    te = []74    cl = []75 76    for name, value in message.headers.fields:77        if not _valid_header_name.match(name):78            raise ValueError(f"invalid header name: {name!r}")79        match name.lower():80            case b"transfer-encoding":81                te.append(value)82            case b"content-length":83                cl.append(value)84 85    if te and cl:86        # > A server MAY reject a request that contains both Content-Length and Transfer-Encoding or process such a87        # > request in accordance with the Transfer-Encoding alone.88 89        # > A sender MUST NOT send a Content-Length header field in any message that contains a Transfer-Encoding header90        # > field.91        raise ValueError(92            "message with both transfer-encoding and content-length headers"93        )94    elif te:95        if len(te) > 1:96            raise ValueError(f"multiple transfer-encoding headers: {te!r}")97        # > Transfer-Encoding was added in HTTP/1.1. It is generally assumed that implementations advertising only98        # > HTTP/1.0 support will not understand how to process transfer-encoded content, and that an HTTP/1.0 message99        # > received with a Transfer-Encoding is likely to have been forwarded without proper handling of the chunked100        # > transfer coding in transit.101        #102        # > A client MUST NOT send a request containing Transfer-Encoding unless it knows the server will handle103        # > HTTP/1.1 requests (or later minor revisions); such knowledge might be in the form of specific user104        # > configuration or by remembering the version of a prior received response. A server MUST NOT send a response105        # > containing Transfer-Encoding unless the corresponding request indicates HTTP/1.1 (or later minor revisions).106        if not message.is_http11:107            raise ValueError(108                f"unexpected HTTP transfer-encoding {te[0]!r} for {message.http_version}"109            )110        # > A server MUST NOT send a Transfer-Encoding header field in any response with a status code of 1xx111        # > (Informational) or 204 (No Content).112        if isinstance(message, Response) and (113            100 <= message.status_code <= 199 or message.status_code == 204114        ):115            raise ValueError(116                f"unexpected HTTP transfer-encoding {te[0]!r} for response with status code {message.status_code}"117            )118        # > If a Transfer-Encoding header field is present in a request and the chunked transfer coding is not the final119        # > encoding, the message body length cannot be determined reliably; the server MUST respond with the 400 (Bad120        # > Request) status code and then close the connection.121        te_parsed = parse_transfer_encoding(te[0])122        match te_parsed:123            case "chunked" | "compress,chunked" | "deflate,chunked" | "gzip,chunked":124                pass125            case "compress" | "deflate" | "gzip" | "identity":126                if isinstance(message, Request):127                    raise ValueError(128                        f"unexpected HTTP transfer-encoding {te_parsed!r} for request"129                    )130            case other:  # pragma: no cover131                typing.assert_never(other)132    elif cl:133        # > If a message is received without Transfer-Encoding and with an invalid Content-Length header field, then the134        # > message framing is invalid and the recipient MUST treat it as an unrecoverable error, unless the field value135        # > can be successfully parsed as a comma-separated list (Section 5.6.1 of [HTTP]), all values in the list are136        # > valid, and all values in the list are the same (in which case, the message is processed with that single137        # > value used as the Content-Length field value).138        # We are stricter here and reject comma-separated lists.139        if len(cl) > 1:140            raise ValueError(f"multiple content-length headers: {cl!r}")141        parse_content_length(cl[0])142