codekingpro/portable-devtools

1# Code to read HTTP data2#3# Strategy: each reader is a callable which takes a ReceiveBuffer object, and4# either:5# 1) consumes some of it and returns an Event6# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate()7#    and it might raise a LocalProtocolError, so simpler just to always use8#    this)9# 3) returns None, meaning "I need more data"10#11# If they have a .read_eof attribute, then this will be called if an EOF is12# received -- but this is optional. Either way, the actual ConnectionClosed13# event will be generated afterwards.14#15# READERS is a dict describing how to pick a reader. It maps states to either:16# - a reader17# - or, for body readers, a dict of per-framing reader factories18 19import re20from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union21 22from ._abnf import chunk_header, header_field, request_line, status_line23from ._events import Data, EndOfMessage, InformationalResponse, Request, Response24from ._receivebuffer import ReceiveBuffer25from ._state import (26    CLIENT,27    CLOSED,28    DONE,29    IDLE,30    MUST_CLOSE,31    SEND_BODY,32    SEND_RESPONSE,33    SERVER,34)35from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate36 37__all__ = ["READERS"]38 39header_field_re = re.compile(header_field.encode("ascii"))40obs_fold_re = re.compile(rb"[ \t]+")41 42 43def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]:44    it = iter(lines)45    last: Optional[bytes] = None46    for line in it:47        match = obs_fold_re.match(line)48        if match:49            if last is None:50                raise LocalProtocolError("continuation line at start of headers")51            if not isinstance(last, bytearray):52                # Cast to a mutable type, avoiding copy on append to ensure O(n) time53                last = bytearray(last)54            last += b" "55            last += line[match.end() :]56        else:57            if last is not None:58                yield last59            last = line60    if last is not None:61        yield last62 63 64def _decode_header_lines(65    lines: Iterable[bytes],66) -> Iterable[Tuple[bytes, bytes]]:67    for line in _obsolete_line_fold(lines):68        matches = validate(header_field_re, line, "illegal header line: {!r}", line)69        yield (matches["field_name"], matches["field_value"])70 71 72request_line_re = re.compile(request_line.encode("ascii"))73 74 75def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]:76    lines = buf.maybe_extract_lines()77    if lines is None:78        if buf.is_next_line_obviously_invalid_request_line():79            raise LocalProtocolError("illegal request line")80        return None81    if not lines:82        raise LocalProtocolError("no request line received")83    matches = validate(84        request_line_re, lines[0], "illegal request line: {!r}", lines[0]85    )86    return Request(87        headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches88    )89 90 91status_line_re = re.compile(status_line.encode("ascii"))92 93 94def maybe_read_from_SEND_RESPONSE_server(95    buf: ReceiveBuffer,96) -> Union[InformationalResponse, Response, None]:97    lines = buf.maybe_extract_lines()98    if lines is None:99        if buf.is_next_line_obviously_invalid_request_line():100            raise LocalProtocolError("illegal request line")101        return None102    if not lines:103        raise LocalProtocolError("no response line received")104    matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0])105    http_version = (106        b"1.1" if matches["http_version"] is None else matches["http_version"]107    )108    reason = b"" if matches["reason"] is None else matches["reason"]109    status_code = int(matches["status_code"])110    class_: Union[Type[InformationalResponse], Type[Response]] = (111        InformationalResponse if status_code < 200 else Response112    )113    return class_(114        headers=list(_decode_header_lines(lines[1:])),115        _parsed=True,116        status_code=status_code,117        reason=reason,118        http_version=http_version,119    )120 121 122class ContentLengthReader:123    def __init__(self, length: int) -> None:124        self._length = length125        self._remaining = length126 127    def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]:128        if self._remaining == 0:129            return EndOfMessage()130        data = buf.maybe_extract_at_most(self._remaining)131        if data is None:132            return None133        self._remaining -= len(data)134        return Data(data=data)135 136    def read_eof(self) -> NoReturn:137        raise RemoteProtocolError(138            "peer closed connection without sending complete message body "139            "(received {} bytes, expected {})".format(140                self._length - self._remaining, self._length141            )142        )143 144 145chunk_header_re = re.compile(chunk_header.encode("ascii"))146 147 148class ChunkedReader:149    def __init__(self) -> None:150        self._bytes_in_chunk = 0151        # After reading a chunk, we have to throw away the trailing \r\n; if152        # this is >0 then we discard that many bytes before resuming regular153        # de-chunkification.154        self._bytes_to_discard = 0155        self._reading_trailer = False156 157    def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]:158        if self._reading_trailer:159            lines = buf.maybe_extract_lines()160            if lines is None:161                return None162            return EndOfMessage(headers=list(_decode_header_lines(lines)))163        if self._bytes_to_discard > 0:164            data = buf.maybe_extract_at_most(self._bytes_to_discard)165            if data is None:166                return None167            self._bytes_to_discard -= len(data)168            if self._bytes_to_discard > 0:169                return None170            # else, fall through and read some more171        assert self._bytes_to_discard == 0172        if self._bytes_in_chunk == 0:173            # We need to refill our chunk count174            chunk_header = buf.maybe_extract_next_line()175            if chunk_header is None:176                return None177            matches = validate(178                chunk_header_re,179                chunk_header,180                "illegal chunk header: {!r}",181                chunk_header,182            )183            # XX FIXME: we discard chunk extensions. Does anyone care?184            self._bytes_in_chunk = int(matches["chunk_size"], base=16)185            if self._bytes_in_chunk == 0:186                self._reading_trailer = True187                return self(buf)188            chunk_start = True189        else:190            chunk_start = False191        assert self._bytes_in_chunk > 0192        data = buf.maybe_extract_at_most(self._bytes_in_chunk)193        if data is None:194            return None195        self._bytes_in_chunk -= len(data)196        if self._bytes_in_chunk == 0:197            self._bytes_to_discard = 2198            chunk_end = True199        else:200            chunk_end = False201        return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end)202 203    def read_eof(self) -> NoReturn:204        raise RemoteProtocolError(205            "peer closed connection without sending complete message body "206            "(incomplete chunked read)"207        )208 209 210class Http10Reader:211    def __call__(self, buf: ReceiveBuffer) -> Optional[Data]:212        data = buf.maybe_extract_at_most(999999999)213        if data is None:214            return None215        return Data(data=data)216 217    def read_eof(self) -> EndOfMessage:218        return EndOfMessage()219 220 221def expect_nothing(buf: ReceiveBuffer) -> None:222    if buf:223        raise LocalProtocolError("Got data when expecting EOF")224    return None225 226 227ReadersType = Dict[228    Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]],229    Union[Callable[..., Any], Dict[str, Callable[..., Any]]],230]231 232READERS: ReadersType = {233    (CLIENT, IDLE): maybe_read_from_IDLE_client,234    (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server,235    (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server,236    (CLIENT, DONE): expect_nothing,237    (CLIENT, MUST_CLOSE): expect_nothing,238    (CLIENT, CLOSED): expect_nothing,239    (SERVER, DONE): expect_nothing,240    (SERVER, MUST_CLOSE): expect_nothing,241    (SERVER, CLOSED): expect_nothing,242    SEND_BODY: {243        "chunked": ChunkedReader,244        "content-length": ContentLengthReader,245        "http/1.0": Http10Reader,246    },247}248