codekingpro/portable-devtools

1# This contains the main Connection class. Everything in h11 revolves around2# this.3from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Type, Union4 5from ._events import (6    ConnectionClosed,7    Data,8    EndOfMessage,9    Event,10    InformationalResponse,11    Request,12    Response,13)14from ._headers import get_comma_header, has_expect_100_continue, set_comma_header15from ._readers import READERS, ReadersType16from ._receivebuffer import ReceiveBuffer17from ._state import (18    _SWITCH_CONNECT,19    _SWITCH_UPGRADE,20    CLIENT,21    ConnectionState,22    DONE,23    ERROR,24    MIGHT_SWITCH_PROTOCOL,25    SEND_BODY,26    SERVER,27    SWITCHED_PROTOCOL,28)29from ._util import (  # Import the internal things we need30    LocalProtocolError,31    RemoteProtocolError,32    Sentinel,33)34from ._writers import WRITERS, WritersType35 36# Everything in __all__ gets re-exported as part of the h11 public API.37__all__ = ["Connection", "NEED_DATA", "PAUSED"]38 39 40class NEED_DATA(Sentinel, metaclass=Sentinel):41    pass42 43 44class PAUSED(Sentinel, metaclass=Sentinel):45    pass46 47 48# If we ever have this much buffered without it making a complete parseable49# event, we error out. The only time we really buffer is when reading the50# request/response line + headers together, so this is effectively the limit on51# the size of that.52#53# Some precedents for defaults:54# - node.js: 80 * 102455# - tomcat: 8 * 102456# - IIS: 16 * 102457# - Apache: <8 KiB per line>58DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 102459 60# RFC 7230's rules for connection lifecycles:61# - If either side says they want to close the connection, then the connection62#   must close.63# - HTTP/1.1 defaults to keep-alive unless someone says Connection: close64# - HTTP/1.0 defaults to close unless both sides say Connection: keep-alive65#   (and even this is a mess -- e.g. if you're implementing a proxy then66#   sending Connection: keep-alive is forbidden).67#68# We simplify life by simply not supporting keep-alive with HTTP/1.0 peers. So69# our rule is:70# - If someone says Connection: close, we will close71# - If someone uses HTTP/1.0, we will close.72def _keep_alive(event: Union[Request, Response]) -> bool:73    connection = get_comma_header(event.headers, b"connection")74    if b"close" in connection:75        return False76    if getattr(event, "http_version", b"1.1") < b"1.1":77        return False78    return True79 80 81def _body_framing(82    request_method: bytes, event: Union[Request, Response]83) -> Tuple[str, Union[Tuple[()], Tuple[int]]]:84    # Called when we enter SEND_BODY to figure out framing information for85    # this body.86    #87    # These are the only two events that can trigger a SEND_BODY state:88    assert type(event) in (Request, Response)89    # Returns one of:90    #91    #    ("content-length", count)92    #    ("chunked", ())93    #    ("http/1.0", ())94    #95    # which are (lookup key, *args) for constructing body reader/writer96    # objects.97    #98    # Reference: https://tools.ietf.org/html/rfc7230#section-3.3.399    #100    # Step 1: some responses always have an empty body, regardless of what the101    # headers say.102    if type(event) is Response:103        if (104            event.status_code in (204, 304)105            or request_method == b"HEAD"106            or (request_method == b"CONNECT" and 200 <= event.status_code < 300)107        ):108            return ("content-length", (0,))109        # Section 3.3.3 also lists another case -- responses with status_code110        # < 200. For us these are InformationalResponses, not Responses, so111        # they can't get into this function in the first place.112        assert event.status_code >= 200113 114    # Step 2: check for Transfer-Encoding (T-E beats C-L):115    transfer_encodings = get_comma_header(event.headers, b"transfer-encoding")116    if transfer_encodings:117        assert transfer_encodings == [b"chunked"]118        return ("chunked", ())119 120    # Step 3: check for Content-Length121    content_lengths = get_comma_header(event.headers, b"content-length")122    if content_lengths:123        return ("content-length", (int(content_lengths[0]),))124 125    # Step 4: no applicable headers; fallback/default depends on type126    if type(event) is Request:127        return ("content-length", (0,))128    else:129        return ("http/1.0", ())130 131 132################################################################133#134# The main Connection class135#136################################################################137 138 139class Connection:140    """An object encapsulating the state of an HTTP connection.141 142    Args:143        our_role: If you're implementing a client, pass :data:`h11.CLIENT`. If144            you're implementing a server, pass :data:`h11.SERVER`.145 146        max_incomplete_event_size (int):147            The maximum number of bytes we're willing to buffer of an148            incomplete event. In practice this mostly sets a limit on the149            maximum size of the request/response line + headers. If this is150            exceeded, then :meth:`next_event` will raise151            :exc:`RemoteProtocolError`.152 153    """154 155    def __init__(156        self,157        our_role: Type[Sentinel],158        max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE,159    ) -> None:160        self._max_incomplete_event_size = max_incomplete_event_size161        # State and role tracking162        if our_role not in (CLIENT, SERVER):163            raise ValueError("expected CLIENT or SERVER, not {!r}".format(our_role))164        self.our_role = our_role165        self.their_role: Type[Sentinel]166        if our_role is CLIENT:167            self.their_role = SERVER168        else:169            self.their_role = CLIENT170        self._cstate = ConnectionState()171 172        # Callables for converting data->events or vice-versa given the173        # current state174        self._writer = self._get_io_object(self.our_role, None, WRITERS)175        self._reader = self._get_io_object(self.their_role, None, READERS)176 177        # Holds any unprocessed received data178        self._receive_buffer = ReceiveBuffer()179        # If this is true, then it indicates that the incoming connection was180        # closed *after* the end of whatever's in self._receive_buffer:181        self._receive_buffer_closed = False182 183        # Extra bits of state that don't fit into the state machine.184        #185        # These two are only used to interpret framing headers for figuring186        # out how to read/write response bodies. their_http_version is also187        # made available as a convenient public API.188        self.their_http_version: Optional[bytes] = None189        self._request_method: Optional[bytes] = None190        # This is pure flow-control and doesn't at all affect the set of legal191        # transitions, so no need to bother ConnectionState with it:192        self.client_is_waiting_for_100_continue = False193 194    @property195    def states(self) -> Dict[Type[Sentinel], Type[Sentinel]]:196        """A dictionary like::197 198           {CLIENT: <client state>, SERVER: <server state>}199 200        See :ref:`state-machine` for details.201 202        """203        return dict(self._cstate.states)204 205    @property206    def our_state(self) -> Type[Sentinel]:207        """The current state of whichever role we are playing. See208        :ref:`state-machine` for details.209        """210        return self._cstate.states[self.our_role]211 212    @property213    def their_state(self) -> Type[Sentinel]:214        """The current state of whichever role we are NOT playing. See215        :ref:`state-machine` for details.216        """217        return self._cstate.states[self.their_role]218 219    @property220    def they_are_waiting_for_100_continue(self) -> bool:221        return self.their_role is CLIENT and self.client_is_waiting_for_100_continue222 223    def start_next_cycle(self) -> None:224        """Attempt to reset our connection state for a new request/response225        cycle.226 227        If both client and server are in :data:`DONE` state, then resets them228        both to :data:`IDLE` state in preparation for a new request/response229        cycle on this same connection. Otherwise, raises a230        :exc:`LocalProtocolError`.231 232        See :ref:`keepalive-and-pipelining`.233 234        """235        old_states = dict(self._cstate.states)236        self._cstate.start_next_cycle()237        self._request_method = None238        # self.their_http_version gets left alone, since it presumably lasts239        # beyond a single request/response cycle240        assert not self.client_is_waiting_for_100_continue241        self._respond_to_state_changes(old_states)242 243    def _process_error(self, role: Type[Sentinel]) -> None:244        old_states = dict(self._cstate.states)245        self._cstate.process_error(role)246        self._respond_to_state_changes(old_states)247 248    def _server_switch_event(self, event: Event) -> Optional[Type[Sentinel]]:249        if type(event) is InformationalResponse and event.status_code == 101:250            return _SWITCH_UPGRADE251        if type(event) is Response:252            if (253                _SWITCH_CONNECT in self._cstate.pending_switch_proposals254                and 200 <= event.status_code < 300255            ):256                return _SWITCH_CONNECT257        return None258 259    # All events go through here260    def _process_event(self, role: Type[Sentinel], event: Event) -> None:261        # First, pass the event through the state machine to make sure it262        # succeeds.263        old_states = dict(self._cstate.states)264        if role is CLIENT and type(event) is Request:265            if event.method == b"CONNECT":266                self._cstate.process_client_switch_proposal(_SWITCH_CONNECT)267            if get_comma_header(event.headers, b"upgrade"):268                self._cstate.process_client_switch_proposal(_SWITCH_UPGRADE)269        server_switch_event = None270        if role is SERVER:271            server_switch_event = self._server_switch_event(event)272        self._cstate.process_event(role, type(event), server_switch_event)273 274        # Then perform the updates triggered by it.275 276        if type(event) is Request:277            self._request_method = event.method278 279        if role is self.their_role and type(event) in (280            Request,281            Response,282            InformationalResponse,283        ):284            event = cast(Union[Request, Response, InformationalResponse], event)285            self.their_http_version = event.http_version286 287        # Keep alive handling288        #289        # RFC 7230 doesn't really say what one should do if Connection: close290        # shows up on a 1xx InformationalResponse. I think the idea is that291        # this is not supposed to happen. In any case, if it does happen, we292        # ignore it.293        if type(event) in (Request, Response) and not _keep_alive(294            cast(Union[Request, Response], event)295        ):296            self._cstate.process_keep_alive_disabled()297 298        # 100-continue299        if type(event) is Request and has_expect_100_continue(event):300            self.client_is_waiting_for_100_continue = True301        if type(event) in (InformationalResponse, Response):302            self.client_is_waiting_for_100_continue = False303        if role is CLIENT and type(event) in (Data, EndOfMessage):304            self.client_is_waiting_for_100_continue = False305 306        self._respond_to_state_changes(old_states, event)307 308    def _get_io_object(309        self,310        role: Type[Sentinel],311        event: Optional[Event],312        io_dict: Union[ReadersType, WritersType],313    ) -> Optional[Callable[..., Any]]:314        # event may be None; it's only used when entering SEND_BODY315        state = self._cstate.states[role]316        if state is SEND_BODY:317            # Special case: the io_dict has a dict of reader/writer factories318            # that depend on the request/response framing.319            framing_type, args = _body_framing(320                cast(bytes, self._request_method), cast(Union[Request, Response], event)321            )322            return io_dict[SEND_BODY][framing_type](*args)  # type: ignore[index]323        else:324            # General case: the io_dict just has the appropriate reader/writer325            # for this state326            return io_dict.get((role, state))  # type: ignore[return-value]327 328    # This must be called after any action that might have caused329    # self._cstate.states to change.330    def _respond_to_state_changes(331        self,332        old_states: Dict[Type[Sentinel], Type[Sentinel]],333        event: Optional[Event] = None,334    ) -> None:335        # Update reader/writer336        if self.our_state != old_states[self.our_role]:337            self._writer = self._get_io_object(self.our_role, event, WRITERS)338        if self.their_state != old_states[self.their_role]:339            self._reader = self._get_io_object(self.their_role, event, READERS)340 341    @property342    def trailing_data(self) -> Tuple[bytes, bool]:343        """Data that has been received, but not yet processed, represented as344        a tuple with two elements, where the first is a byte-string containing345        the unprocessed data itself, and the second is a bool that is True if346        the receive connection was closed.347 348        See :ref:`switching-protocols` for discussion of why you'd want this.349        """350        return (bytes(self._receive_buffer), self._receive_buffer_closed)351 352    def receive_data(self, data: bytes) -> None:353        """Add data to our internal receive buffer.354 355        This does not actually do any processing on the data, just stores356        it. To trigger processing, you have to call :meth:`next_event`.357 358        Args:359            data (:term:`bytes-like object`):360                The new data that was just received.361 362                Special case: If *data* is an empty byte-string like ``b""``,363                then this indicates that the remote side has closed the364                connection (end of file). Normally this is convenient, because365                standard Python APIs like :meth:`file.read` or366                :meth:`socket.recv` use ``b""`` to indicate end-of-file, while367                other failures to read are indicated using other mechanisms368                like raising :exc:`TimeoutError`. When using such an API you369                can just blindly pass through whatever you get from ``read``370                to :meth:`receive_data`, and everything will work.371 372                But, if you have an API where reading an empty string is a373                valid non-EOF condition, then you need to be aware of this and374                make sure to check for such strings and avoid passing them to375                :meth:`receive_data`.376 377        Returns:378            Nothing, but after calling this you should call :meth:`next_event`379            to parse the newly received data.380 381        Raises:382            RuntimeError:383                Raised if you pass an empty *data*, indicating EOF, and then384                pass a non-empty *data*, indicating more data that somehow385                arrived after the EOF.386 387                (Calling ``receive_data(b"")`` multiple times is fine,388                and equivalent to calling it once.)389 390        """391        if data:392            if self._receive_buffer_closed:393                raise RuntimeError("received close, then received more data?")394            self._receive_buffer += data395        else:396            self._receive_buffer_closed = True397 398    def _extract_next_receive_event(399        self,400    ) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]:401        state = self.their_state402        # We don't pause immediately when they enter DONE, because even in403        # DONE state we can still process a ConnectionClosed() event. But404        # if we have data in our buffer, then we definitely aren't getting405        # a ConnectionClosed() immediately and we need to pause.406        if state is DONE and self._receive_buffer:407            return PAUSED408        if state is MIGHT_SWITCH_PROTOCOL or state is SWITCHED_PROTOCOL:409            return PAUSED410        assert self._reader is not None411        event = self._reader(self._receive_buffer)412        if event is None:413            if not self._receive_buffer and self._receive_buffer_closed:414                # In some unusual cases (basically just HTTP/1.0 bodies), EOF415                # triggers an actual protocol event; in that case, we want to416                # return that event, and then the state will change and we'll417                # get called again to generate the actual ConnectionClosed().418                if hasattr(self._reader, "read_eof"):419                    event = self._reader.read_eof()  # type: ignore[attr-defined]420                else:421                    event = ConnectionClosed()422        if event is None:423            event = NEED_DATA424        return event  # type: ignore[no-any-return]425 426    def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]:427        """Parse the next event out of our receive buffer, update our internal428        state, and return it.429 430        This is a mutating operation -- think of it like calling :func:`next`431        on an iterator.432 433        Returns:434            : One of three things:435 436            1) An event object -- see :ref:`events`.437 438            2) The special constant :data:`NEED_DATA`, which indicates that439               you need to read more data from your socket and pass it to440               :meth:`receive_data` before this method will be able to return441               any more events.442 443            3) The special constant :data:`PAUSED`, which indicates that we444               are not in a state where we can process incoming data (usually445               because the peer has finished their part of the current446               request/response cycle, and you have not yet called447               :meth:`start_next_cycle`). See :ref:`flow-control` for details.448 449        Raises:450            RemoteProtocolError:451                The peer has misbehaved. You should close the connection452                (possibly after sending some kind of 4xx response).453 454        Once this method returns :class:`ConnectionClosed` once, then all455        subsequent calls will also return :class:`ConnectionClosed`.456 457        If this method raises any exception besides :exc:`RemoteProtocolError`458        then that's a bug -- if it happens please file a bug report!459 460        If this method raises any exception then it also sets461        :attr:`Connection.their_state` to :data:`ERROR` -- see462        :ref:`error-handling` for discussion.463 464        """465 466        if self.their_state is ERROR:467            raise RemoteProtocolError("Can't receive data when peer state is ERROR")468        try:469            event = self._extract_next_receive_event()470            if event not in [NEED_DATA, PAUSED]:471                self._process_event(self.their_role, cast(Event, event))472            if event is NEED_DATA:473                if len(self._receive_buffer) > self._max_incomplete_event_size:474                    # 431 is "Request header fields too large" which is pretty475                    # much the only situation where we can get here476                    raise RemoteProtocolError(477                        "Receive buffer too long", error_status_hint=431478                    )479                if self._receive_buffer_closed:480                    # We're still trying to complete some event, but that's481                    # never going to happen because no more data is coming482                    raise RemoteProtocolError("peer unexpectedly closed connection")483            return event484        except BaseException as exc:485            self._process_error(self.their_role)486            if isinstance(exc, LocalProtocolError):487                exc._reraise_as_remote_protocol_error()488            else:489                raise490 491    def send(self, event: Event) -> Optional[bytes]:492        """Convert a high-level event into bytes that can be sent to the peer,493        while updating our internal state machine.494 495        Args:496            event: The :ref:`event <events>` to send.497 498        Returns:499            If ``type(event) is ConnectionClosed``, then returns500            ``None``. Otherwise, returns a :term:`bytes-like object`.501 502        Raises:503            LocalProtocolError:504                Sending this event at this time would violate our505                understanding of the HTTP/1.1 protocol.506 507        If this method raises any exception then it also sets508        :attr:`Connection.our_state` to :data:`ERROR` -- see509        :ref:`error-handling` for discussion.510 511        """512        data_list = self.send_with_data_passthrough(event)513        if data_list is None:514            return None515        else:516            return b"".join(data_list)517 518    def send_with_data_passthrough(self, event: Event) -> Optional[List[bytes]]:519        """Identical to :meth:`send`, except that in situations where520        :meth:`send` returns a single :term:`bytes-like object`, this instead521        returns a list of them -- and when sending a :class:`Data` event, this522        list is guaranteed to contain the exact object you passed in as523        :attr:`Data.data`. See :ref:`sendfile` for discussion.524 525        """526        if self.our_state is ERROR:527            raise LocalProtocolError("Can't send data when our state is ERROR")528        try:529            if type(event) is Response:530                event = self._clean_up_response_headers_for_sending(event)531            # We want to call _process_event before calling the writer,532            # because if someone tries to do something invalid then this will533            # give a sensible error message, while our writers all just assume534            # they will only receive valid events. But, _process_event might535            # change self._writer. So we have to do a little dance:536            writer = self._writer537            self._process_event(self.our_role, event)538            if type(event) is ConnectionClosed:539                return None540            else:541                # In any situation where writer is None, process_event should542                # have raised ProtocolError543                assert writer is not None544                data_list: List[bytes] = []545                writer(event, data_list.append)546                return data_list547        except:548            self._process_error(self.our_role)549            raise550 551    def send_failed(self) -> None:552        """Notify the state machine that we failed to send the data it gave553        us.554 555        This causes :attr:`Connection.our_state` to immediately become556        :data:`ERROR` -- see :ref:`error-handling` for discussion.557 558        """559        self._process_error(self.our_role)560 561    # When sending a Response, we take responsibility for a few things:562    #563    # - Sometimes you MUST set Connection: close. We take care of those564    #   times. (You can also set it yourself if you want, and if you do then565    #   we'll respect that and close the connection at the right time. But you566    #   don't have to worry about that unless you want to.)567    #568    # - The user has to set Content-Length if they want it. Otherwise, for569    #   responses that have bodies (e.g. not HEAD), then we will automatically570    #   select the right mechanism for streaming a body of unknown length,571    #   which depends on depending on the peer's HTTP version.572    #573    # This function's *only* responsibility is making sure headers are set up574    # right -- everything downstream just looks at the headers. There are no575    # side channels.576    def _clean_up_response_headers_for_sending(self, response: Response) -> Response:577        assert type(response) is Response578 579        headers = response.headers580        need_close = False581 582        # HEAD requests need some special handling: they always act like they583        # have Content-Length: 0, and that's how _body_framing treats584        # them. But their headers are supposed to match what we would send if585        # the request was a GET. (Technically there is one deviation allowed:586        # we're allowed to leave out the framing headers -- see587        # https://tools.ietf.org/html/rfc7231#section-4.3.2 . But it's just as588        # easy to get them right.)589        method_for_choosing_headers = cast(bytes, self._request_method)590        if method_for_choosing_headers == b"HEAD":591            method_for_choosing_headers = b"GET"592        framing_type, _ = _body_framing(method_for_choosing_headers, response)593        if framing_type in ("chunked", "http/1.0"):594            # This response has a body of unknown length.595            # If our peer is HTTP/1.1, we use Transfer-Encoding: chunked596            # If our peer is HTTP/1.0, we use no framing headers, and close the597            # connection afterwards.598            #599            # Make sure to clear Content-Length (in principle user could have600            # set both and then we ignored Content-Length b/c601            # Transfer-Encoding overwrote it -- this would be naughty of them,602            # but the HTTP spec says that if our peer does this then we have603            # to fix it instead of erroring out, so we'll accord the user the604            # same respect).605            headers = set_comma_header(headers, b"content-length", [])606            if self.their_http_version is None or self.their_http_version < b"1.1":607                # Either we never got a valid request and are sending back an608                # error (their_http_version is None), so we assume the worst;609                # or else we did get a valid HTTP/1.0 request, so we know that610                # they don't understand chunked encoding.611                headers = set_comma_header(headers, b"transfer-encoding", [])612                # This is actually redundant ATM, since currently we613                # unconditionally disable keep-alive when talking to HTTP/1.0614                # peers. But let's be defensive just in case we add615                # Connection: keep-alive support later:616                if self._request_method != b"HEAD":617                    need_close = True618            else:619                headers = set_comma_header(headers, b"transfer-encoding", [b"chunked"])620 621        if not self._cstate.keep_alive or need_close:622            # Make sure Connection: close is set623            connection = set(get_comma_header(headers, b"connection"))624            connection.discard(b"keep-alive")625            connection.add(b"close")626            headers = set_comma_header(headers, b"connection", sorted(connection))627 628        return Response(629            headers=headers,630            status_code=response.status_code,631            http_version=response.http_version,632            reason=response.reason,633        )634