codekingpro/portable-devtools

1import struct2import time3import typing4from collections.abc import Iterator5from dataclasses import dataclass6from logging import DEBUG7from logging import ERROR8from logging import INFO9from logging import WARNING10 11from OpenSSL import SSL12 13from mitmproxy import certs14from mitmproxy import connection15from mitmproxy.connection import TlsVersion16from mitmproxy.net.tls import starts_like_dtls_record17from mitmproxy.net.tls import starts_like_tls_record18from mitmproxy.proxy import commands19from mitmproxy.proxy import context20from mitmproxy.proxy import events21from mitmproxy.proxy import layer22from mitmproxy.proxy import tunnel23from mitmproxy.proxy.commands import StartHook24from mitmproxy.proxy.layers import tcp25from mitmproxy.proxy.layers import udp26from mitmproxy.tls import ClientHello27from mitmproxy.tls import ClientHelloData28from mitmproxy.tls import TlsData29from mitmproxy.utils import human30 31 32def handshake_record_contents(data: bytes) -> Iterator[bytes]:33    """34    Returns a generator that yields the bytes contained in each handshake record.35    This will raise an error on the first non-handshake record, so fully exhausting this36    generator is a bad idea.37    """38    offset = 039    while True:40        if len(data) < offset + 5:41            return42        record_header = data[offset : offset + 5]43        if not starts_like_tls_record(record_header):44            raise ValueError(f"Expected TLS record, got {record_header!r} instead.")45        record_size = struct.unpack("!H", record_header[3:])[0]46        if record_size == 0:47            raise ValueError("Record must not be empty.")48        offset += 549 50        if len(data) < offset + record_size:51            return52        record_body = data[offset : offset + record_size]53        yield record_body54        offset += record_size55 56 57def get_client_hello(data: bytes) -> bytes | None:58    """59    Read all TLS records that contain the initial ClientHello.60    Returns the raw handshake packet bytes, without TLS record headers.61    """62    client_hello = b""63    for d in handshake_record_contents(data):64        client_hello += d65        if len(client_hello) >= 4:66            client_hello_size = struct.unpack("!I", b"\x00" + client_hello[1:4])[0] + 467            if len(client_hello) >= client_hello_size:68                return client_hello[:client_hello_size]69    return None70 71 72def parse_client_hello(data: bytes) -> ClientHello | None:73    """74    Check if the supplied bytes contain a full ClientHello message,75    and if so, parse it.76 77    Returns:78        - A ClientHello object on success79        - None, if the TLS record is not complete80 81    Raises:82        - A ValueError, if the passed ClientHello is invalid83    """84    # Check if ClientHello is complete85    client_hello = get_client_hello(data)86    if client_hello:87        try:88            return ClientHello(client_hello[4:])89        except EOFError as e:90            raise ValueError("Invalid ClientHello") from e91    return None92 93 94def dtls_handshake_record_contents(data: bytes) -> Iterator[bytes]:95    """96    Returns a generator that yields the bytes contained in each handshake record.97    This will raise an error on the first non-handshake record, so fully exhausting this98    generator is a bad idea.99    """100    offset = 0101    while True:102        # DTLS includes two new fields, totaling 8 bytes, between Version and Length103        if len(data) < offset + 13:104            return105        record_header = data[offset : offset + 13]106        if not starts_like_dtls_record(record_header):107            raise ValueError(f"Expected DTLS record, got {record_header!r} instead.")108        # Length fields starts at 11109        record_size = struct.unpack("!H", record_header[11:])[0]110        if record_size == 0:111            raise ValueError("Record must not be empty.")112        offset += 13113 114        if len(data) < offset + record_size:115            return116        record_body = data[offset : offset + record_size]117        yield record_body118        offset += record_size119 120 121def get_dtls_client_hello(data: bytes) -> bytes | None:122    """123    Read all DTLS records that contain the initial ClientHello.124    Returns the raw handshake packet bytes, without TLS record headers.125    """126    client_hello = b""127    for d in dtls_handshake_record_contents(data):128        client_hello += d129        if len(client_hello) >= 13:130            # comment about slicing: we skip the epoch and sequence number131            client_hello_size = (132                struct.unpack("!I", b"\x00" + client_hello[9:12])[0] + 12133            )134            if len(client_hello) >= client_hello_size:135                return client_hello[:client_hello_size]136    return None137 138 139def dtls_parse_client_hello(data: bytes) -> ClientHello | None:140    """141    Check if the supplied bytes contain a full ClientHello message,142    and if so, parse it.143 144    Returns:145        - A ClientHello object on success146        - None, if the TLS record is not complete147 148    Raises:149        - A ValueError, if the passed ClientHello is invalid150    """151    # Check if ClientHello is complete152    client_hello = get_dtls_client_hello(data)153    if client_hello:154        try:155            return ClientHello(client_hello[12:], dtls=True)156        except EOFError as e:157            raise ValueError("Invalid ClientHello") from e158    return None159 160 161HTTP1_ALPNS = (b"http/1.1", b"http/1.0", b"http/0.9")162HTTP2_ALPN = b"h2"163HTTP3_ALPN = b"h3"164HTTP_ALPNS = (HTTP3_ALPN, HTTP2_ALPN, *HTTP1_ALPNS)165 166 167# We need these classes as hooks can only have one argument at the moment.168 169 170@dataclass171class TlsClienthelloHook(StartHook):172    """173    Mitmproxy has received a TLS ClientHello message.174 175    This hook decides whether a server connection is needed176    to negotiate TLS with the client (data.establish_server_tls_first)177    """178 179    data: ClientHelloData180 181 182@dataclass183class TlsStartClientHook(StartHook):184    """185    TLS negotation between mitmproxy and a client is about to start.186 187    An addon is expected to initialize data.ssl_conn.188    (by default, this is done by `mitmproxy.addons.tlsconfig`)189    """190 191    data: TlsData192 193 194@dataclass195class TlsStartServerHook(StartHook):196    """197    TLS negotation between mitmproxy and a server is about to start.198 199    An addon is expected to initialize data.ssl_conn.200    (by default, this is done by `mitmproxy.addons.tlsconfig`)201    """202 203    data: TlsData204 205 206@dataclass207class TlsEstablishedClientHook(StartHook):208    """209    The TLS handshake with the client has been completed successfully.210    """211 212    data: TlsData213 214 215@dataclass216class TlsEstablishedServerHook(StartHook):217    """218    The TLS handshake with the server has been completed successfully.219    """220 221    data: TlsData222 223 224@dataclass225class TlsFailedClientHook(StartHook):226    """227    The TLS handshake with the client has failed.228    """229 230    data: TlsData231 232 233@dataclass234class TlsFailedServerHook(StartHook):235    """236    The TLS handshake with the server has failed.237    """238 239    data: TlsData240 241 242class TLSLayer(tunnel.TunnelLayer):243    tls: SSL.Connection = None  # type: ignore244    """The OpenSSL connection object"""245 246    def __init__(self, context: context.Context, conn: connection.Connection):247        super().__init__(248            context,249            tunnel_connection=conn,250            conn=conn,251        )252 253        conn.tls = True254 255    def __repr__(self):256        return (257            super().__repr__().replace(")", f" {self.conn.sni!r} {self.conn.alpn!r})")258        )259 260    @property261    def is_dtls(self):262        return self.conn.transport_protocol == "udp"263 264    @property265    def proto_name(self):266        return "DTLS" if self.is_dtls else "TLS"267 268    def start_tls(self) -> layer.CommandGenerator[None]:269        assert not self.tls270 271        tls_start = TlsData(self.conn, self.context, is_dtls=self.is_dtls)272        if self.conn == self.context.client:273            yield TlsStartClientHook(tls_start)274        else:275            yield TlsStartServerHook(tls_start)276        if not tls_start.ssl_conn:277            yield commands.Log(278                f"No {self.proto_name} context was provided, failing connection.", ERROR279            )280            yield commands.CloseConnection(self.conn)281            return282        assert tls_start.ssl_conn283        self.tls = tls_start.ssl_conn284 285    def tls_interact(self) -> layer.CommandGenerator[None]:286        while True:287            try:288                data = self.tls.bio_read(65535)289            except SSL.WantReadError:290                return  # Okay, nothing more waiting to be sent.291            else:292                yield commands.SendData(self.conn, data)293 294    def receive_handshake_data(295        self, data: bytes296    ) -> layer.CommandGenerator[tuple[bool, str | None]]:297        # bio_write errors for b"", so we need to check first if we actually received something.298        if data:299            self.tls.bio_write(data)300        try:301            self.tls.do_handshake()302        except SSL.WantReadError:303            yield from self.tls_interact()304            return False, None305        except SSL.Error as e:306            # provide more detailed information for some errors.307            last_err = (308                e.args and isinstance(e.args[0], list) and e.args[0] and e.args[0][-1]309            )310            if last_err in [311                (312                    "SSL routines",313                    "tls_process_server_certificate",314                    "certificate verify failed",315                ),316                ("SSL routines", "", "certificate verify failed"),  # OpenSSL 3+317            ]:318                verify_result = SSL._lib.SSL_get_verify_result(self.tls._ssl)  # type: ignore319                error = SSL._ffi.string(  # type: ignore320                    SSL._lib.X509_verify_cert_error_string(verify_result)  # type: ignore321                ).decode()322                err = f"Certificate verify failed: {error}"323            elif last_err in [324                ("SSL routines", "ssl3_read_bytes", "tlsv1 alert unknown ca"),325                ("SSL routines", "ssl3_read_bytes", "sslv3 alert bad certificate"),326                ("SSL routines", "ssl3_read_bytes", "ssl/tls alert bad certificate"),327                ("SSL routines", "", "tlsv1 alert unknown ca"),  # OpenSSL 3+328                ("SSL routines", "", "sslv3 alert bad certificate"),  # OpenSSL 3+329                ("SSL routines", "", "ssl/tls alert bad certificate"),  # OpenSSL 3.2+330            ]:331                assert isinstance(last_err, tuple)332                err = last_err[2]333            elif (334                last_err335                in [336                    ("SSL routines", "ssl3_get_record", "wrong version number"),337                    ("SSL routines", "", "wrong version number"),  # OpenSSL 3+338                    ("SSL routines", "", "packet length too long"),  # OpenSSL 3+339                    ("SSL routines", "", "record layer failure"),  # OpenSSL 3+340                ]341                and data[:4].isascii()342            ):343                err = f"The remote server does not speak TLS."344            elif last_err in [345                ("SSL routines", "ssl3_read_bytes", "tlsv1 alert protocol version"),346                ("SSL routines", "", "tlsv1 alert protocol version"),  # OpenSSL 3+347            ]:348                err = (349                    f"The remote server and mitmproxy cannot agree on a TLS version to use. "350                    f"You may need to adjust mitmproxy's tls_version_server_min option."351                )352            else:353                err = f"OpenSSL {e!r}"354            return False, err355        else:356            # Here we set all attributes that are only known *after* the handshake.357 358            # Get all peer certificates.359            # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_cert_chain.html360            # If called on the client side, the stack also contains the peer's certificate; if called on the server361            # side, the peer's certificate must be obtained separately using SSL_get_peer_certificate(3).362            all_certs = self.tls.get_peer_cert_chain() or []363            if self.conn == self.context.client:364                cert = self.tls.get_peer_certificate()365                if cert:366                    all_certs.insert(0, cert)367            self.conn.certificate_list = []368            for cert in all_certs:369                try:370                    # This may fail for weird certs, https://github.com/mitmproxy/mitmproxy/issues/6968.371                    parsed_cert = certs.Cert.from_pyopenssl(cert)372                except ValueError as e:373                    yield commands.Log(374                        f"{self.debug}[tls] failed to parse certificate: {e}", WARNING375                    )376                else:377                    self.conn.certificate_list.append(parsed_cert)378 379            self.conn.timestamp_tls_setup = time.time()380            self.conn.alpn = self.tls.get_alpn_proto_negotiated()381            self.conn.cipher = self.tls.get_cipher_name()382            self.conn.tls_version = typing.cast(383                TlsVersion, self.tls.get_protocol_version_name()384            )385            if self.debug:386                yield commands.Log(387                    f"{self.debug}[tls] tls established: {self.conn}", DEBUG388                )389            if self.conn == self.context.client:390                yield TlsEstablishedClientHook(391                    TlsData(self.conn, self.context, self.tls)392                )393            else:394                yield TlsEstablishedServerHook(395                    TlsData(self.conn, self.context, self.tls)396                )397            yield from self.receive_data(b"")398            return True, None399 400    def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:401        self.conn.error = err402        if self.conn == self.context.client:403            yield TlsFailedClientHook(TlsData(self.conn, self.context, self.tls))404        else:405            yield TlsFailedServerHook(TlsData(self.conn, self.context, self.tls))406        yield from super().on_handshake_error(err)407 408    def receive_data(self, data: bytes) -> layer.CommandGenerator[None]:409        if data:410            self.tls.bio_write(data)411 412        plaintext = bytearray()413        close = False414        while True:415            try:416                plaintext.extend(self.tls.recv(65535))417            except SSL.WantReadError:418                break419            except SSL.ZeroReturnError:420                close = True421                break422            except SSL.Error as e:423                # This may be happening because the other side send an alert.424                # There's somewhat ugly behavior with Firefox on Android here,425                # which upon mistrusting a certificate still completes the handshake426                # and then sends an alert in the next packet. At this point we have unfortunately427                # already fired out `tls_established_client` hook.428                yield commands.Log(f"TLS Error: {e}", WARNING)429                break430 431        # Can we send something?432        # Note that this must happen after `recv()`, which may have advanced the state machine.433        # https://github.com/mitmproxy/mitmproxy/discussions/7550434        yield from self.tls_interact()435 436        if plaintext:437            yield from self.event_to_child(438                events.DataReceived(self.conn, bytes(plaintext))439            )440        if close:441            self.conn.state &= ~connection.ConnectionState.CAN_READ442            if self.debug:443                yield commands.Log(f"{self.debug}[tls] close_notify {self.conn}", DEBUG)444            yield from self.event_to_child(events.ConnectionClosed(self.conn))445 446    def receive_close(self) -> layer.CommandGenerator[None]:447        if self.tls.get_shutdown() & SSL.RECEIVED_SHUTDOWN:448            pass  # We have already dispatched a ConnectionClosed to the child layer.449        else:450            yield from super().receive_close()451 452    def send_data(self, data: bytes) -> layer.CommandGenerator[None]:453        try:454            self.tls.sendall(data)455        except (SSL.ZeroReturnError, SSL.SysCallError):456            # The other peer may still be trying to send data over, which we discard here.457            pass458        yield from self.tls_interact()459 460    def send_close(461        self, command: commands.CloseConnection462    ) -> layer.CommandGenerator[None]:463        # We should probably shutdown the TLS connection properly here.464        yield from super().send_close(command)465 466 467class ServerTLSLayer(TLSLayer):468    """469    This layer establishes TLS for a single server connection.470    """471 472    wait_for_clienthello: bool = False473 474    def __init__(self, context: context.Context, conn: connection.Server | None = None):475        super().__init__(context, conn or context.server)476 477    def start_handshake(self) -> layer.CommandGenerator[None]:478        wait_for_clienthello = (479            # if command_to_reply_to is set, we've been instructed to open the connection from the child layer.480            # in that case any potential ClientHello is already parsed (by the ClientTLS child layer).481            not self.command_to_reply_to482            # if command_to_reply_to is not set, the connection was already open when this layer received its Start483            # event (eager connection strategy). We now want to establish TLS right away, _unless_ we already know484            # that there's TLS on the client side as well (we check if our immediate child layer is set to be ClientTLS)485            # In this case want to wait for ClientHello to be parsed, so that we can incorporate SNI/ALPN from there.486            and isinstance(self.child_layer, ClientTLSLayer)487        )488        if wait_for_clienthello:489            self.wait_for_clienthello = True490            self.tunnel_state = tunnel.TunnelState.CLOSED491        else:492            yield from self.start_tls()493            if self.tls:494                yield from self.receive_handshake_data(b"")495 496    def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]:497        if self.wait_for_clienthello:498            for command in super().event_to_child(event):499                if (500                    isinstance(command, commands.OpenConnection)501                    and command.connection == self.conn502                ):503                    self.wait_for_clienthello = False504                    # swallow OpenConnection here by not re-yielding it.505                else:506                    yield command507        else:508            yield from super().event_to_child(event)509 510    def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:511        yield commands.Log(f"Server TLS handshake failed. {err}", level=WARNING)512        yield from super().on_handshake_error(err)513 514 515class ClientTLSLayer(TLSLayer):516    """517    This layer establishes TLS on a single client connection.518 519    ┌─────┐520    │Start│521    └┬────┘522     ↓523    ┌────────────────────┐524    │Wait for ClientHello│525    └┬───────────────────┘526     ↓527    ┌────────────────┐528    │Process messages│529    └────────────────┘530 531    """532 533    recv_buffer: bytearray534    server_tls_available: bool535    client_hello_parsed: bool = False536 537    def __init__(self, context: context.Context):538        if context.client.tls:539            # In the case of TLS-over-TLS, we already have client TLS. As the outer TLS connection between client540            # and proxy isn't that interesting to us, we just unset the attributes here and keep the inner TLS541            # session's attributes.542            # Alternatively we could create a new Client instance,543            # but for now we keep it simple. There is a proof-of-concept at544            # https://github.com/mitmproxy/mitmproxy/commit/9b6e2a716888b7787514733b76a5936afa485352.545            context.client.alpn = None546            context.client.cipher = None547            context.client.sni = None548            context.client.timestamp_tls_setup = None549            context.client.tls_version = None550            context.client.certificate_list = []551            context.client.mitmcert = None552            context.client.alpn_offers = []553            context.client.cipher_list = []554 555        super().__init__(context, context.client)556        self.server_tls_available = isinstance(self.context.layers[-2], ServerTLSLayer)557        self.recv_buffer = bytearray()558 559    def start_handshake(self) -> layer.CommandGenerator[None]:560        yield from ()561 562    def receive_handshake_data(563        self, data: bytes564    ) -> layer.CommandGenerator[tuple[bool, str | None]]:565        if self.client_hello_parsed:566            return (yield from super().receive_handshake_data(data))567        self.recv_buffer.extend(data)568        try:569            if self.is_dtls:570                client_hello = dtls_parse_client_hello(self.recv_buffer)571            else:572                client_hello = parse_client_hello(self.recv_buffer)573        except ValueError:574            return False, f"Cannot parse ClientHello: {self.recv_buffer.hex()}"575 576        if client_hello:577            self.client_hello_parsed = True578        else:579            return False, None580 581        self.conn.sni = client_hello.sni582        self.conn.alpn_offers = client_hello.alpn_protocols583        tls_clienthello = ClientHelloData(self.context, client_hello)584        yield TlsClienthelloHook(tls_clienthello)585 586        if tls_clienthello.ignore_connection:587            # we've figured out that we don't want to intercept this connection, so we assign fake connection objects588            # to all TLS layers. This makes the real connection contents just go through.589            self.conn = self.tunnel_connection = connection.Client(590                peername=("ignore-conn", 0), sockname=("ignore-conn", 0)591            )592            parent_layer = self.context.layers[self.context.layers.index(self) - 1]593            if isinstance(parent_layer, ServerTLSLayer):594                parent_layer.conn = parent_layer.tunnel_connection = connection.Server(595                    address=None596                )597            if self.is_dtls:598                self.child_layer = udp.UDPLayer(self.context, ignore=True)599            else:600                self.child_layer = tcp.TCPLayer(self.context, ignore=True)601            yield from self.event_to_child(602                events.DataReceived(self.context.client, bytes(self.recv_buffer))603            )604            self.recv_buffer.clear()605            return True, None606        if (607            tls_clienthello.establish_server_tls_first608            and not self.context.server.tls_established609        ):610            err = yield from self.start_server_tls()611            if err:612                yield commands.Log(613                    f"Unable to establish {self.proto_name} connection with server ({err}). "614                    f"Trying to establish {self.proto_name} with client anyway. "615                    f"If you plan to redirect requests away from this server, "616                    f"consider setting `connection_strategy` to `lazy` to suppress early connections."617                )618 619        yield from self.start_tls()620        if not self.conn.connected:621            return False, "connection closed early"622 623        ret = yield from super().receive_handshake_data(bytes(self.recv_buffer))624        self.recv_buffer.clear()625        return ret626 627    def start_server_tls(self) -> layer.CommandGenerator[str | None]:628        """629        We often need information from the upstream connection to establish TLS with the client.630        For example, we need to check if the client does ALPN or not.631        """632        if not self.server_tls_available:633            return f"No server {self.proto_name} available."634        err = yield commands.OpenConnection(self.context.server)635        return err636 637    def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:638        if self.conn.sni:639            dest = self.conn.sni640        else:641            dest = human.format_address(self.context.server.address)642        level: int = WARNING643        if err.startswith("Cannot parse ClientHello"):644            pass645        elif (646            "('SSL routines', 'tls_early_post_process_client_hello', 'unsupported protocol')"647            in err648            or "('SSL routines', '', 'unsupported protocol')" in err  # OpenSSL 3+649        ):650            err = (651                f"Client and mitmproxy cannot agree on a TLS version to use. "652                f"You may need to adjust mitmproxy's tls_version_client_min option."653            )654        elif (655            "unknown ca" in err656            or "bad certificate" in err657            or "certificate unknown" in err658        ):659            err = (660                f"The client does not trust the proxy's certificate for {dest} ({err})"661            )662        elif err == "connection closed":663            err = (664                f"The client disconnected during the handshake. If this happens consistently for {dest}, "665                f"this may indicate that the client does not trust the proxy's certificate."666            )667            level = INFO668        elif err == "connection closed early":669            pass670        else:671            err = f"The client may not trust the proxy's certificate for {dest} ({err})"672        if err != "connection closed early":673            yield commands.Log(f"Client TLS handshake failed. {err}", level=level)674        yield from super().on_handshake_error(err)675        self.event_to_child = self.errored  # type: ignore676 677    def errored(self, event: events.Event) -> layer.CommandGenerator[None]:678        if self.debug is not None:679            yield commands.Log(680                f"{self.debug}[tls] Swallowing {event} as handshake failed.", DEBUG681            )682 683 684class MockTLSLayer(TLSLayer):685    """Mock layer to disable actual TLS and use cleartext in tests.686 687    Use like so:688        monkeypatch.setattr(tls, "ServerTLSLayer", tls.MockTLSLayer)689    """690 691    def __init__(self, ctx: context.Context):692        super().__init__(ctx, connection.Server(address=None))693