codekingpro/portable-devtools

1from cpython.exc cimport PyErr_NoMemory2from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc3from cpython.unicode cimport (4    PyUnicode_DATA,5    PyUnicode_DecodeASCII,6    PyUnicode_DecodeUTF8Stateful,7    PyUnicode_GET_LENGTH,8    PyUnicode_KIND,9    PyUnicode_READ,10)11from libc.stdint cimport uint8_t, uint64_t12from libc.string cimport memcpy, memset13 14from string import ascii_letters, digits15 16 17cdef str GEN_DELIMS = ":/?#[]@"18cdef str SUB_DELIMS_WITHOUT_QS = "!$'()*,"19cdef str SUB_DELIMS = SUB_DELIMS_WITHOUT_QS + '+?=;'20cdef str RESERVED = GEN_DELIMS + SUB_DELIMS21cdef str UNRESERVED = ascii_letters + digits + '-._~'22cdef str ALLOWED = UNRESERVED + SUB_DELIMS_WITHOUT_QS23cdef str QS = '+&=;'24 25DEF BUF_SIZE = 8 * 1024  # 8KiB26 27cdef inline Py_UCS4 _to_hex(uint8_t v) noexcept:28    if v < 10:29        return <Py_UCS4>(v+0x30)  # ord('0') == 0x3030    else:31        return <Py_UCS4>(v+0x41-10)  # ord('A') == 0x4132 33 34cdef inline int _from_hex(Py_UCS4 v) noexcept:35    if '0' <= v <= '9':36        return <int>(v) - 0x30  # ord('0') == 0x3037    elif 'A' <= v <= 'F':38        return <int>(v) - 0x41 + 10  # ord('A') == 0x4139    elif 'a' <= v <= 'f':40        return <int>(v) - 0x61 + 10  # ord('a') == 0x6141    else:42        return -143 44 45cdef inline int _is_lower_hex(Py_UCS4 v) noexcept:46    return 'a' <= v <= 'f'47 48 49cdef inline long _restore_ch(Py_UCS4 d1, Py_UCS4 d2):50    cdef int digit1 = _from_hex(d1)51    if digit1 < 0:52        return -153    cdef int digit2 = _from_hex(d2)54    if digit2 < 0:55        return -156    return digit1 << 4 | digit257 58 59cdef uint8_t ALLOWED_TABLE[16]60cdef uint8_t ALLOWED_NOTQS_TABLE[16]61 62 63cdef inline bint bit_at(uint8_t array[], uint64_t ch) noexcept:64    return array[ch >> 3] & (1 << (ch & 7))65 66 67cdef inline void set_bit(uint8_t array[], uint64_t ch) noexcept:68    array[ch >> 3] |= (1 << (ch & 7))69 70 71memset(ALLOWED_TABLE, 0, sizeof(ALLOWED_TABLE))72memset(ALLOWED_NOTQS_TABLE, 0, sizeof(ALLOWED_NOTQS_TABLE))73 74for i in range(128):75    if chr(i) in ALLOWED:76        set_bit(ALLOWED_TABLE, i)77        set_bit(ALLOWED_NOTQS_TABLE, i)78    if chr(i) in QS:79        set_bit(ALLOWED_NOTQS_TABLE, i)80 81# ----------------- writer ---------------------------82 83cdef struct Writer:84    char *buf85    bint heap_allocated_buf86    Py_ssize_t size87    Py_ssize_t pos88    bint changed89 90 91cdef inline void _init_writer(Writer* writer, char* buf):92    writer.buf = buf93    writer.heap_allocated_buf = False94    writer.size = BUF_SIZE95    writer.pos = 096    writer.changed = 097 98 99cdef inline void _release_writer(Writer* writer):100    if writer.heap_allocated_buf:101        PyMem_Free(writer.buf)102 103 104cdef inline int _write_char(Writer* writer, Py_UCS4 ch, bint changed):105    cdef char * buf106    cdef Py_ssize_t size107 108    if writer.pos == writer.size:109        # reallocate110        size = writer.size + BUF_SIZE111        if not writer.heap_allocated_buf:112            buf = <char*>PyMem_Malloc(size)113            if buf == NULL:114                PyErr_NoMemory()115                return -1116            memcpy(buf, writer.buf, writer.size)117            writer.heap_allocated_buf = True118        else:119            buf = <char*>PyMem_Realloc(writer.buf, size)120            if buf == NULL:121                PyErr_NoMemory()122                return -1123        writer.buf = buf124        writer.size = size125    writer.buf[writer.pos] = <char>ch126    writer.pos += 1127    writer.changed |= changed128    return 0129 130 131cdef inline int _write_pct(Writer* writer, uint8_t ch, bint changed):132    if _write_char(writer, '%', changed) < 0:133        return -1134    if _write_char(writer, _to_hex(<uint8_t>ch >> 4), changed) < 0:135        return -1136    return _write_char(writer, _to_hex(<uint8_t>ch & 0x0f), changed)137 138 139cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol):140    cdef uint64_t utf = <uint64_t> symbol141 142    if utf < 0x80:143        return _write_pct(writer, <uint8_t>utf, True)144    elif utf < 0x800:145        if _write_pct(writer, <uint8_t>(0xc0 | (utf >> 6)), True) < 0:146            return -1147        return _write_pct(writer,  <uint8_t>(0x80 | (utf & 0x3f)), True)148    elif 0xD800 <= utf <= 0xDFFF:149        # surogate pair, ignored150        return 0151    elif utf < 0x10000:152        if _write_pct(writer, <uint8_t>(0xe0 | (utf >> 12)), True) < 0:153            return -1154        if _write_pct(writer, <uint8_t>(0x80 | ((utf >> 6) & 0x3f)),155                      True) < 0:156            return -1157        return _write_pct(writer, <uint8_t>(0x80 | (utf & 0x3f)), True)158    elif utf > 0x10FFFF:159        # symbol is too large160        return 0161    else:162        if _write_pct(writer,  <uint8_t>(0xf0 | (utf >> 18)), True) < 0:163            return -1164        if _write_pct(writer,  <uint8_t>(0x80 | ((utf >> 12) & 0x3f)),165                      True) < 0:166            return -1167        if _write_pct(writer,  <uint8_t>(0x80 | ((utf >> 6) & 0x3f)),168                      True) < 0:169            return -1170        return _write_pct(writer, <uint8_t>(0x80 | (utf & 0x3f)), True)171 172 173# --------------------- end writer --------------------------174 175 176cdef class _Quoter:177    cdef bint _qs178    cdef bint _requote179 180    cdef uint8_t _safe_table[16]181    cdef uint8_t _protected_table[16]182 183    def __init__(184            self, *, str safe='', str protected='', bint qs=False, bint requote=True,185    ):186        cdef Py_UCS4 ch187 188        self._qs = qs189        self._requote = requote190 191        if not self._qs:192            memcpy(self._safe_table,193                   ALLOWED_NOTQS_TABLE,194                   sizeof(self._safe_table))195        else:196            memcpy(self._safe_table,197                   ALLOWED_TABLE,198                   sizeof(self._safe_table))199        for ch in safe:200            if ord(ch) > 127:201                raise ValueError("Only safe symbols with ORD < 128 are allowed")202            set_bit(self._safe_table, ch)203 204        memset(self._protected_table, 0, sizeof(self._protected_table))205        for ch in protected:206            if ord(ch) > 127:207                raise ValueError("Only safe symbols with ORD < 128 are allowed")208            set_bit(self._safe_table, ch)209            set_bit(self._protected_table, ch)210 211    def __call__(self, val):212        if val is None:213            return None214        if type(val) is not str:215            if isinstance(val, str):216                # derived from str217                val = str(val)218            else:219                raise TypeError("Argument should be str")220        return self._do_quote_or_skip(<str>val)221 222    cdef str _do_quote_or_skip(self, str val):223        cdef char[BUF_SIZE] buffer224        cdef Py_UCS4 ch225        cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val)226        cdef Py_ssize_t idx = length227        cdef bint must_quote = 0228        cdef Writer writer229        cdef int kind = PyUnicode_KIND(val)230        cdef const void *data = PyUnicode_DATA(val)231 232        # If everything in the string is in the safe233        # table and all ASCII, we can skip quoting234        while idx:235            idx -= 1236            ch = PyUnicode_READ(kind, data, idx)237            if ch >= 128 or not bit_at(self._safe_table, ch):238                must_quote = 1239                break240 241        if not must_quote:242            return val243 244        _init_writer(&writer, &buffer[0])245        try:246            return self._do_quote(<str>val, length, kind, data, &writer)247        finally:248            _release_writer(&writer)249 250    cdef str _do_quote(251        self,252        str val,253        Py_ssize_t length,254        int kind,255        const void *data,256        Writer *writer257    ):258        cdef Py_UCS4 ch259        cdef long chl260        cdef int changed261        cdef Py_ssize_t idx = 0262 263        while idx < length:264            ch = PyUnicode_READ(kind, data, idx)265            idx += 1266            if ch == '%' and self._requote and idx <= length - 2:267                chl = _restore_ch(268                    PyUnicode_READ(kind, data, idx),269                    PyUnicode_READ(kind, data, idx + 1)270                )271                if chl != -1:272                    ch = <Py_UCS4>chl273                    idx += 2274                    if ch < 128:275                        if bit_at(self._protected_table, ch):276                            if _write_pct(writer, ch, True) < 0:277                                raise278                            continue279 280                        if bit_at(self._safe_table, ch):281                            if _write_char(writer, ch, True) < 0:282                                raise283                            continue284 285                    changed = (_is_lower_hex(PyUnicode_READ(kind, data, idx - 2)) or286                               _is_lower_hex(PyUnicode_READ(kind, data, idx - 1)))287                    if _write_pct(writer, ch, changed) < 0:288                        raise289                    continue290                else:291                    ch = '%'292 293            if self._write(writer, ch) < 0:294                raise295 296        if not writer.changed:297            return val298        else:299            return PyUnicode_DecodeASCII(writer.buf, writer.pos, "strict")300 301    cdef inline int _write(self, Writer *writer, Py_UCS4 ch):302        if self._qs:303            if ch == ' ':304                return _write_char(writer, '+', True)305 306        if ch < 128 and bit_at(self._safe_table, ch):307            return _write_char(writer, ch, False)308 309        return _write_utf8(writer, ch)310 311 312cdef class _Unquoter:313    cdef str _ignore314    cdef bint _has_ignore315    cdef str _unsafe316    cdef bytes _unsafe_bytes317    cdef Py_ssize_t _unsafe_bytes_len318    cdef const unsigned char * _unsafe_bytes_char319    cdef bint _qs320    cdef bint _plus  # to match urllib.parse.unquote_plus321    cdef _Quoter _quoter322    cdef _Quoter _qs_quoter323 324    def __init__(self, *, ignore="", unsafe="", qs=False, plus=False):325        self._ignore = ignore326        self._has_ignore = bool(self._ignore)327        self._unsafe = unsafe328        # unsafe may only be extended ascii characters (0-255)329        self._unsafe_bytes = self._unsafe.encode('ascii')330        self._unsafe_bytes_len = len(self._unsafe_bytes)331        self._unsafe_bytes_char = self._unsafe_bytes332        self._qs = qs333        self._plus = plus334        self._quoter = _Quoter()335        self._qs_quoter = _Quoter(qs=True)336 337    def __call__(self, val):338        if val is None:339            return None340        if type(val) is not str:341            if isinstance(val, str):342                # derived from str343                val = str(val)344            else:345                raise TypeError("Argument should be str")346        return self._do_unquote(<str>val)347 348    cdef str _do_unquote(self, str val):349        cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val)350        if length == 0:351            return val352 353        cdef list ret = []354        cdef char buffer[4]355        cdef Py_ssize_t buflen = 0356        cdef Py_ssize_t consumed357        cdef str unquoted358        cdef Py_UCS4 ch = 0359        cdef long chl = 0360        cdef Py_ssize_t idx = 0361        cdef Py_ssize_t start_pct362        cdef int kind = PyUnicode_KIND(val)363        cdef const void *data = PyUnicode_DATA(val)364        cdef bint changed = 0365        while idx < length:366            ch = PyUnicode_READ(kind, data, idx)367            idx += 1368            if ch == '%' and idx <= length - 2:369                changed = 1370                chl = _restore_ch(371                    PyUnicode_READ(kind, data, idx),372                    PyUnicode_READ(kind, data, idx + 1)373                )374                if chl != -1:375                    ch = <Py_UCS4>chl376                    idx += 2377                    assert buflen < 4378                    buffer[buflen] = ch379                    buflen += 1380                    try:381                        unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen,382                                                                NULL, &consumed)383                    except UnicodeDecodeError:384                        start_pct = idx - buflen * 3385                        buffer[0] = ch386                        buflen = 1387                        ret.append(val[start_pct : idx - 3])388                        try:389                            unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen,390                                                                    NULL, &consumed)391                        except UnicodeDecodeError:392                            buflen = 0393                            ret.append(val[idx - 3 : idx])394                            continue395                    if not unquoted:396                        assert consumed == 0397                        continue398                    assert consumed == buflen399                    buflen = 0400                    if self._qs and unquoted in '+=&;':401                        ret.append(self._qs_quoter(unquoted))402                    elif (403                        (self._unsafe_bytes_len and unquoted in self._unsafe) or404                        (self._has_ignore and unquoted in self._ignore)405                    ):406                        ret.append(self._quoter(unquoted))407                    else:408                        ret.append(unquoted)409                    continue410                else:411                    ch = '%'412 413            if buflen:414                start_pct = idx - 1 - buflen * 3415                ret.append(val[start_pct : idx - 1])416                buflen = 0417 418            if ch == '+':419                if (420                    (not self._qs and not self._plus) or421                    (self._unsafe_bytes_len and self._is_char_unsafe(ch))422                ):423                    ret.append('+')424                else:425                    changed = 1426                    ret.append(' ')427                continue428 429            if self._unsafe_bytes_len and self._is_char_unsafe(ch):430                changed = 1431                ret.append('%')432                h = hex(ord(ch)).upper()[2:]433                for ch in h:434                    ret.append(ch)435                continue436 437            ret.append(ch)438 439        if not changed:440            return val441 442        if buflen:443            ret.append(val[length - buflen * 3 : length])444 445        return ''.join(ret)446 447    cdef inline bint _is_char_unsafe(self, Py_UCS4 ch):448        for i in range(self._unsafe_bytes_len):449            if ch == self._unsafe_bytes_char[i]:450                return True451        return False452