codekingpro/portable-devtools

1import abc2from collections.abc import Callable3from typing import Union4 5import h116from h11._readers import ChunkedReader7from h11._readers import ContentLengthReader8from h11._readers import Http10Reader9from h11._receivebuffer import ReceiveBuffer10 11from ...context import Context12from ._base import format_error13from ._base import HttpConnection14from ._events import ErrorCode15from ._events import HttpEvent16from ._events import RequestData17from ._events import RequestEndOfMessage18from ._events import RequestHeaders19from ._events import RequestProtocolError20from ._events import ResponseData21from ._events import ResponseEndOfMessage22from ._events import ResponseHeaders23from ._events import ResponseProtocolError24from mitmproxy import http25from mitmproxy import version26from mitmproxy.connection import Connection27from mitmproxy.connection import ConnectionState28from mitmproxy.net.http import http129from mitmproxy.net.http import status_codes30from mitmproxy.proxy import commands31from mitmproxy.proxy import events32from mitmproxy.proxy import layer33from mitmproxy.proxy.layers.http._base import ReceiveHttp34from mitmproxy.proxy.layers.http._base import StreamId35from mitmproxy.proxy.utils import expect36from mitmproxy.utils import human37 38TBodyReader = Union[ChunkedReader, Http10Reader, ContentLengthReader]39 40 41class Http1Connection(HttpConnection, metaclass=abc.ABCMeta):42    stream_id: StreamId | None = None43    request: http.Request | None = None44    response: http.Response | None = None45    request_done: bool = False46    response_done: bool = False47    # this is a bit of a hack to make both mypy and PyCharm happy.48    state: Callable[[events.Event], layer.CommandGenerator[None]] | Callable49    body_reader: TBodyReader50    buf: ReceiveBuffer51 52    ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError]53    ReceiveData: type[RequestData | ResponseData]54    ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage]55 56    def __init__(self, context: Context, conn: Connection):57        super().__init__(context, conn)58        self.buf = ReceiveBuffer()59 60    @abc.abstractmethod61    def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:62        yield from ()  # pragma: no cover63 64    @abc.abstractmethod65    def read_headers(66        self, event: events.ConnectionEvent67    ) -> layer.CommandGenerator[None]:68        yield from ()  # pragma: no cover69 70    def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]:71        if isinstance(event, HttpEvent):72            yield from self.send(event)73        else:74            if (75                isinstance(event, events.DataReceived)76                and self.state != self.passthrough77            ):78                self.buf += event.data79            yield from self.state(event)80 81    @expect(events.Start)82    def start(self, _) -> layer.CommandGenerator[None]:83        self.state = self.read_headers84        yield from ()85 86    state = start87 88    def read_body(self, event: events.Event) -> layer.CommandGenerator[None]:89        assert self.stream_id is not None90        while True:91            try:92                if isinstance(event, events.DataReceived):93                    h11_event = self.body_reader(self.buf)94                elif isinstance(event, events.ConnectionClosed):95                    h11_event = self.body_reader.read_eof()96                else:97                    raise AssertionError(f"Unexpected event: {event}")98            except h11.ProtocolError as e:99                yield commands.CloseConnection(self.conn)100                yield ReceiveHttp(101                    self.ReceiveProtocolError(102                        self.stream_id,103                        f"HTTP/1 protocol error: {e}",104                        code=self.ReceiveProtocolError.code,105                    )106                )107                return108 109            if h11_event is None:110                return111            elif isinstance(h11_event, h11.Data):112                data: bytes = bytes(h11_event.data)113                if data:114                    yield ReceiveHttp(self.ReceiveData(self.stream_id, data))115            elif isinstance(h11_event, h11.EndOfMessage):116                assert self.request117                if h11_event.headers:118                    raise NotImplementedError(f"HTTP trailers are not implemented yet.")119                if self.request.data.method.upper() != b"CONNECT":120                    yield ReceiveHttp(self.ReceiveEndOfMessage(self.stream_id))121                is_request = isinstance(self, Http1Server)122                yield from self.mark_done(request=is_request, response=not is_request)123                return124 125    def wait(self, event: events.Event) -> layer.CommandGenerator[None]:126        """127        We wait for the current flow to be finished before parsing the next message,128        as we may want to upgrade to WebSocket or plain TCP before that.129        """130        assert self.stream_id131        if isinstance(event, events.DataReceived):132            return133        elif isinstance(event, events.ConnectionClosed):134            # for practical purposes, we assume that a peer which sent at least a FIN135            # is not interested in any more data from us, see136            # see https://github.com/httpwg/http-core/issues/22137            if event.connection.state is not ConnectionState.CLOSED:138                yield commands.CloseConnection(event.connection)139            yield ReceiveHttp(140                self.ReceiveProtocolError(141                    self.stream_id,142                    f"Client disconnected.",143                    code=ErrorCode.CLIENT_DISCONNECTED,144                )145            )146        else:  # pragma: no cover147            raise AssertionError(f"Unexpected event: {event}")148 149    def done(self, event: events.ConnectionEvent) -> layer.CommandGenerator[None]:150        yield from ()  # pragma: no cover151 152    def make_pipe(self) -> layer.CommandGenerator[None]:153        self.state = self.passthrough154        if self.buf:155            already_received = self.buf.maybe_extract_at_most(len(self.buf)) or b""156            # Some clients send superfluous newlines after CONNECT, we want to eat those.157            already_received = already_received.lstrip(b"\r\n")158            if already_received:159                yield from self.state(events.DataReceived(self.conn, already_received))160 161    def passthrough(self, event: events.Event) -> layer.CommandGenerator[None]:162        assert self.stream_id163        if isinstance(event, events.DataReceived):164            yield ReceiveHttp(self.ReceiveData(self.stream_id, event.data))165        elif isinstance(event, events.ConnectionClosed):166            if isinstance(self, Http1Server):167                yield ReceiveHttp(RequestEndOfMessage(self.stream_id))168            else:169                yield ReceiveHttp(ResponseEndOfMessage(self.stream_id))170 171    def mark_done(172        self, *, request: bool = False, response: bool = False173    ) -> layer.CommandGenerator[None]:174        if request:175            self.request_done = True176        if response:177            self.response_done = True178        if self.request_done and self.response_done:179            assert self.request180            assert self.response181            if should_make_pipe(self.request, self.response):182                yield from self.make_pipe()183                return184            try:185                read_until_eof_semantics = (186                    http1.expected_http_body_size(self.request, self.response) == -1187                )188            except ValueError:189                # this may raise only now (and not earlier) because an addon set invalid headers,190                # in which case it's not really clear what we are supposed to do.191                read_until_eof_semantics = False192            connection_done = (193                read_until_eof_semantics194                or http1.connection_close(195                    self.request.http_version, self.request.headers196                )197                or http1.connection_close(198                    self.response.http_version, self.response.headers199                )200                # If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request.201                # This simplifies our connection management quite a bit as we can rely on202                # the proxyserver's max-connection-per-server throttling.203                or (204                    (self.request.is_http2 or self.request.is_http3)205                    and isinstance(self, Http1Client)206                )207            )208            if connection_done:209                yield commands.CloseConnection(self.conn)210                self.state = self.done211                return212            self.request_done = self.response_done = False213            self.request = self.response = None214            if isinstance(self, Http1Server):215                self.stream_id += 2216            else:217                self.stream_id = None218            self.state = self.read_headers219            if self.buf:220                yield from self.state(events.DataReceived(self.conn, b""))221 222 223class Http1Server(Http1Connection):224    """A simple HTTP/1 server with no pipelining support."""225 226    ReceiveProtocolError = RequestProtocolError227    ReceiveData = RequestData228    ReceiveEndOfMessage = RequestEndOfMessage229    stream_id: int230 231    def __init__(self, context: Context):232        super().__init__(context, context.client)233        self.stream_id = 1234 235    def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:236        assert event.stream_id == self.stream_id237        if isinstance(event, ResponseHeaders):238            self.response = response = event.response239 240            if response.is_http2 or response.is_http3:241                response = response.copy()242                # Convert to an HTTP/1 response.243                response.http_version = "HTTP/1.1"244                # not everyone supports empty reason phrases, so we better make up one.245                response.reason = status_codes.RESPONSES.get(response.status_code, "")246                # Shall we set a Content-Length header here if there is none?247                # For now, let's try to modify as little as possible.248 249            raw = http1.assemble_response_head(response)250            yield commands.SendData(self.conn, raw)251        elif isinstance(event, ResponseData):252            assert self.response253            if "chunked" in self.response.headers.get("transfer-encoding", "").lower():254                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)255            else:256                raw = event.data257            if raw:258                yield commands.SendData(self.conn, raw)259        elif isinstance(event, ResponseEndOfMessage):260            assert self.request261            assert self.response262            if (263                self.request.method.upper() != "HEAD"264                and "chunked"265                in self.response.headers.get("transfer-encoding", "").lower()266            ):267                yield commands.SendData(self.conn, b"0\r\n\r\n")268            yield from self.mark_done(response=True)269        elif isinstance(event, ResponseProtocolError):270            if not (self.conn.state & ConnectionState.CAN_WRITE):271                return272            status = event.code.http_status_code()273            if not self.response and status is not None:274                yield commands.SendData(275                    self.conn, make_error_response(status, event.message)276                )277            yield commands.CloseConnection(self.conn)278        else:279            raise AssertionError(f"Unexpected event: {event}")280 281    def read_headers(282        self, event: events.ConnectionEvent283    ) -> layer.CommandGenerator[None]:284        if isinstance(event, events.DataReceived):285            request_head = self.buf.maybe_extract_lines()286            if request_head:287                try:288                    self.request = http1.read_request_head(289                        [bytes(x) for x in request_head]290                    )291                    expected_body_size = http1.expected_http_body_size(self.request)292                except ValueError as e:293                    yield commands.SendData(self.conn, make_error_response(400, str(e)))294                    yield commands.CloseConnection(self.conn)295                    if self.request:296                        # we have headers that we can show in the ui297                        yield ReceiveHttp(298                            RequestHeaders(self.stream_id, self.request, False)299                        )300                        yield ReceiveHttp(301                            RequestProtocolError(302                                self.stream_id, str(e), ErrorCode.GENERIC_CLIENT_ERROR303                            )304                        )305                    else:306                        yield commands.Log(307                            f"{human.format_address(self.conn.peername)}: {e}"308                        )309                    self.state = self.done310                    return311                yield ReceiveHttp(312                    RequestHeaders(313                        self.stream_id, self.request, expected_body_size == 0314                    )315                )316                self.body_reader = make_body_reader(expected_body_size)317                self.state = self.read_body318                yield from self.state(event)319            else:320                pass  # FIXME: protect against header size DoS321        elif isinstance(event, events.ConnectionClosed):322            buf = bytes(self.buf)323            if buf.strip():324                yield commands.Log(325                    f"Client closed connection before completing request headers: {buf!r}"326                )327            yield commands.CloseConnection(self.conn)328        else:329            raise AssertionError(f"Unexpected event: {event}")330 331    def mark_done(332        self, *, request: bool = False, response: bool = False333    ) -> layer.CommandGenerator[None]:334        yield from super().mark_done(request=request, response=response)335        if self.request_done and not self.response_done:336            self.state = self.wait337 338 339class Http1Client(Http1Connection):340    """A simple HTTP/1 client with no pipelining support."""341 342    ReceiveProtocolError = ResponseProtocolError343    ReceiveData = ResponseData344    ReceiveEndOfMessage = ResponseEndOfMessage345 346    def __init__(self, context: Context):347        super().__init__(context, context.server)348 349    def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:350        if isinstance(event, RequestProtocolError):351            yield commands.CloseConnection(self.conn)352            return353 354        if self.stream_id is None:355            assert isinstance(event, RequestHeaders)356            self.stream_id = event.stream_id357            self.request = event.request358        assert self.stream_id == event.stream_id359 360        if isinstance(event, RequestHeaders):361            request = event.request362            if request.is_http2 or request.is_http3:363                # Convert to an HTTP/1 request.364                request = (365                    request.copy()366                )  # (we could probably be a bit more efficient here.)367                request.http_version = "HTTP/1.1"368                if "Host" not in request.headers and request.authority:369                    request.headers.insert(0, "Host", request.authority)370                request.authority = ""371                cookie_headers = request.headers.get_all("Cookie")372                if len(cookie_headers) > 1:373                    # Only HTTP/2 supports multiple cookie headers, HTTP/1.x does not.374                    # see: https://www.rfc-editor.org/rfc/rfc6265#section-5.4375                    #      https://www.rfc-editor.org/rfc/rfc7540#section-8.1.2.5376                    request.headers["Cookie"] = "; ".join(cookie_headers)377            raw = http1.assemble_request_head(request)378            yield commands.SendData(self.conn, raw)379        elif isinstance(event, RequestData):380            assert self.request381            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():382                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)383            else:384                raw = event.data385            if raw:386                yield commands.SendData(self.conn, raw)387        elif isinstance(event, RequestEndOfMessage):388            assert self.request389            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():390                yield commands.SendData(self.conn, b"0\r\n\r\n")391            elif http1.expected_http_body_size(self.request, self.response) == -1:392                yield commands.CloseTcpConnection(self.conn, half_close=True)393            yield from self.mark_done(request=True)394        else:395            raise AssertionError(f"Unexpected event: {event}")396 397    def read_headers(398        self, event: events.ConnectionEvent399    ) -> layer.CommandGenerator[None]:400        if isinstance(event, events.DataReceived):401            if not self.request:402                # we just received some data for an unknown request.403                yield commands.Log(f"Unexpected data from server: {bytes(self.buf)!r}")404                yield commands.CloseConnection(self.conn)405                return406            assert self.stream_id is not None407 408            response_head = self.buf.maybe_extract_lines()409            if response_head:410                try:411                    self.response = http1.read_response_head(412                        [bytes(x) for x in response_head]413                    )414                    expected_size = http1.expected_http_body_size(415                        self.request, self.response416                    )417                except ValueError as e:418                    yield commands.CloseConnection(self.conn)419                    yield ReceiveHttp(420                        ResponseProtocolError(421                            self.stream_id,422                            f"Cannot parse HTTP response: {e}",423                            ErrorCode.GENERIC_SERVER_ERROR,424                        )425                    )426                    return427                yield ReceiveHttp(428                    ResponseHeaders(self.stream_id, self.response, expected_size == 0)429                )430                self.body_reader = make_body_reader(expected_size)431 432                self.state = self.read_body433                yield from self.state(event)434            else:435                pass  # FIXME: protect against header size DoS436        elif isinstance(event, events.ConnectionClosed):437            if self.conn.state & ConnectionState.CAN_WRITE:438                yield commands.CloseConnection(self.conn)439            if self.stream_id:440                if self.buf:441                    yield ReceiveHttp(442                        ResponseProtocolError(443                            self.stream_id,444                            f"unexpected server response: {bytes(self.buf)!r}",445                            ErrorCode.GENERIC_SERVER_ERROR,446                        )447                    )448                else:449                    # The server has closed the connection to prevent us from continuing.450                    # We need to signal that to the stream.451                    # https://tools.ietf.org/html/rfc7231#section-6.5.11452                    yield ReceiveHttp(453                        ResponseProtocolError(454                            self.stream_id,455                            "server closed connection",456                            ErrorCode.GENERIC_SERVER_ERROR,457                        )458                    )459            else:460                return461        else:462            raise AssertionError(f"Unexpected event: {event}")463 464 465def should_make_pipe(request: http.Request, response: http.Response) -> bool:466    if response.status_code == 101:467        return True468    elif response.status_code == 200 and request.method.upper() == "CONNECT":469        return True470    else:471        return False472 473 474def make_body_reader(expected_size: int | None) -> TBodyReader:475    if expected_size is None:476        return ChunkedReader()477    elif expected_size == -1:478        return Http10Reader()479    else:480        return ContentLengthReader(expected_size)481 482 483def make_error_response(484    status_code: int,485    message: str = "",486) -> bytes:487    resp = http.Response.make(488        status_code,489        format_error(status_code, message),490        http.Headers(491            Server=version.MITMPROXY,492            Connection="close",493            Content_Type="text/html",494        ),495    )496    return http1.assemble_response(resp)497 498 499__all__ = [500    "Http1Client",501    "Http1Server",502]503