codekingpro/portable-devtools

1# Code to read HTTP data2#3# Strategy: each reader is a callable which takes a ReceiveBuffer object, and4# either:5# 1) consumes some of it and returns an Event6# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate()7#    and it might raise a LocalProtocolError, so simpler just to always use8#    this)9# 3) returns None, meaning "I need more data"10#11# If they have a .read_eof attribute, then this will be called if an EOF is12# received -- but this is optional. Either way, the actual ConnectionClosed13# event will be generated afterwards.14#15# READERS is a dict describing how to pick a reader. It maps states to either:16# - a reader17# - or, for body readers, a dict of per-framing reader factories18 19import re20from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union21 22from ._abnf import chunk_header, header_field, request_line, status_line23from ._events import Data, EndOfMessage, InformationalResponse, Request, Response24from ._receivebuffer import ReceiveBuffer25from ._state import (26    CLIENT,27    CLOSED,28    DONE,29    IDLE,30    MUST_CLOSE,31    SEND_BODY,32    SEND_RESPONSE,33    SERVER,34)35from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate36 37__all__ = ["READERS"]38 39header_field_re = re.compile(header_field.encode("ascii"))40obs_fold_re = re.compile(rb"[ \t]+")41 42 43def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]:44    it = iter(lines)45    last: Optional[bytes] = None46    for line in it:47        match = obs_fold_re.match(line)48        if match:49            if last is None:50                raise LocalProtocolError("continuation line at start of headers")51            if not isinstance(last, bytearray):52                # Cast to a mutable type, avoiding copy on append to ensure O(n) time53                last = bytearray(last)54            last += b" "55            last += line[match.end() :]56        else:57            if last is not None:58                yield last59            last = line60    if last is not None:61        yield last62 63 64def _decode_header_lines(65    lines: Iterable[bytes],66) -> Iterable[Tuple[bytes, bytes]]:67    for line in _obsolete_line_fold(lines):68        matches = validate(header_field_re, line, "illegal header line: {!r}", line)69        yield (matches["field_name"], matches["field_value"])70 71 72request_line_re = re.compile(request_line.encode("ascii"))73 74 75def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]:76    lines = buf.maybe_extract_lines()77    if lines is None:78        if buf.is_next_line_obviously_invalid_request_line():79            raise LocalProtocolError("illegal request line")80        return None81    if not lines:82        raise LocalProtocolError("no request line received")83    matches = validate(84        request_line_re, lines[0], "illegal request line: {!r}", lines[0]85    )86    return Request(87        headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches88    )89 90 91status_line_re = re.compile(status_line.encode("ascii"))92 93 94def maybe_read_from_SEND_RESPONSE_server(95    buf: ReceiveBuffer,96) -> Union[InformationalResponse, Response, None]:97    lines = buf.maybe_extract_lines()98    if lines is None:99        if buf.is_next_line_obviously_invalid_request_line():100            raise LocalProtocolError("illegal request line")101        return None102    if not lines:103        raise LocalProtocolError("no response line received")104    matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0])105    http_version = (106        b"1.1" if matches["http_version"] is None else matches["http_version"]107    )108    reason = b"" if matches["reason"] is None else matches["reason"]109    status_code = int(matches["status_code"])110    class_: Union[Type[InformationalResponse], Type[Response]] = (111        InformationalResponse if status_code < 200 else Response112    )113    return class_(114        headers=list(_decode_header_lines(lines[1:])),115        _parsed=True,116        status_code=status_code,117        reason=reason,118        http_version=http_version,119    )120 121 122class ContentLengthReader:123    def __init__(self, length: int) -> None:124        self._length = length125        self._remaining = length126 127    def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]:128        if self._remaining == 0:129            return EndOfMessage()130        data = buf.maybe_extract_at_most(self._remaining)131        if data is None:132            return None133        self._remaining -= len(data)134        return Data(data=data)135 136    def read_eof(self) -> NoReturn:137        raise RemoteProtocolError(138            "peer closed connection without sending complete message body "139            "(received {} bytes, expected {})".format(140                self._length - self._remaining, self._length141            )142        )143 144 145chunk_header_re = re.compile(chunk_header.encode("ascii"))146 147 148class ChunkedReader:149    def __init__(self) -> None:150        self._bytes_in_chunk = 0151        # After reading a chunk, we have to throw away the trailing \r\n.152        # This tracks the bytes that we need to match and throw away.153        self._bytes_to_discard = b""154        self._reading_trailer = False155 156    def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]:157        if self._reading_trailer:158            lines = buf.maybe_extract_lines()159            if lines is None:160                return None161            return EndOfMessage(headers=list(_decode_header_lines(lines)))162        if self._bytes_to_discard:163            data = buf.maybe_extract_at_most(len(self._bytes_to_discard))164            if data is None:165                return None166            if data != self._bytes_to_discard[: len(data)]:167                raise LocalProtocolError(168                    f"malformed chunk footer: {data!r} (expected {self._bytes_to_discard!r})"169                )170            self._bytes_to_discard = self._bytes_to_discard[len(data) :]171            if self._bytes_to_discard:172                return None173            # else, fall through and read some more174        assert self._bytes_to_discard == b""175        if self._bytes_in_chunk == 0:176            # We need to refill our chunk count177            chunk_header = buf.maybe_extract_next_line()178            if chunk_header is None:179                return None180            matches = validate(181                chunk_header_re,182                chunk_header,183                "illegal chunk header: {!r}",184                chunk_header,185            )186            # XX FIXME: we discard chunk extensions. Does anyone care?187            self._bytes_in_chunk = int(matches["chunk_size"], base=16)188            if self._bytes_in_chunk == 0:189                self._reading_trailer = True190                return self(buf)191            chunk_start = True192        else:193            chunk_start = False194        assert self._bytes_in_chunk > 0195        data = buf.maybe_extract_at_most(self._bytes_in_chunk)196        if data is None:197            return None198        self._bytes_in_chunk -= len(data)199        if self._bytes_in_chunk == 0:200            self._bytes_to_discard = b"\r\n"201            chunk_end = True202        else:203            chunk_end = False204        return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end)205 206    def read_eof(self) -> NoReturn:207        raise RemoteProtocolError(208            "peer closed connection without sending complete message body "209            "(incomplete chunked read)"210        )211 212 213class Http10Reader:214    def __call__(self, buf: ReceiveBuffer) -> Optional[Data]:215        data = buf.maybe_extract_at_most(999999999)216        if data is None:217            return None218        return Data(data=data)219 220    def read_eof(self) -> EndOfMessage:221        return EndOfMessage()222 223 224def expect_nothing(buf: ReceiveBuffer) -> None:225    if buf:226        raise LocalProtocolError("Got data when expecting EOF")227    return None228 229 230ReadersType = Dict[231    Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]],232    Union[Callable[..., Any], Dict[str, Callable[..., Any]]],233]234 235READERS: ReadersType = {236    (CLIENT, IDLE): maybe_read_from_IDLE_client,237    (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server,238    (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server,239    (CLIENT, DONE): expect_nothing,240    (CLIENT, MUST_CLOSE): expect_nothing,241    (CLIENT, CLOSED): expect_nothing,242    (SERVER, DONE): expect_nothing,243    (SERVER, MUST_CLOSE): expect_nothing,244    (SERVER, CLOSED): expect_nothing,245    SEND_BODY: {246        "chunked": ChunkedReader,247        "content-length": ContentLengthReader,248        "http/1.0": Http10Reader,249    },250}251