codekingpro/portable-devtools

1""" Codec for the Punycode encoding, as specified in RFC 34922 3Written by Martin v. Löwis.4"""5 6import codecs7 8##################### Encoding #####################################9 10def segregate(str):11    """3.1 Basic code point segregation"""12    base = bytearray()13    extended = set()14    for c in str:15        if ord(c) < 128:16            base.append(ord(c))17        else:18            extended.add(c)19    extended = sorted(extended)20    return bytes(base), extended21 22def selective_len(str, max):23    """Return the length of str, considering only characters below max."""24    res = 025    for c in str:26        if ord(c) < max:27            res += 128    return res29 30def selective_find(str, char, index, pos):31    """Return a pair (index, pos), indicating the next occurrence of32    char in str. index is the position of the character considering33    only ordinals up to and including char, and pos is the position in34    the full string. index/pos is the starting position in the full35    string."""36 37    l = len(str)38    while 1:39        pos += 140        if pos == l:41            return (-1, -1)42        c = str[pos]43        if c == char:44            return index+1, pos45        elif c < char:46            index += 147 48def insertion_unsort(str, extended):49    """3.2 Insertion unsort coding"""50    oldchar = 0x8051    result = []52    oldindex = -153    for c in extended:54        index = pos = -155        char = ord(c)56        curlen = selective_len(str, char)57        delta = (curlen+1) * (char - oldchar)58        while 1:59            index,pos = selective_find(str,c,index,pos)60            if index == -1:61                break62            delta += index - oldindex63            result.append(delta-1)64            oldindex = index65            delta = 066        oldchar = char67 68    return result69 70def T(j, bias):71    # Punycode parameters: tmin = 1, tmax = 26, base = 3672    res = 36 * (j + 1) - bias73    if res < 1: return 174    if res > 26: return 2675    return res76 77digits = b"abcdefghijklmnopqrstuvwxyz0123456789"78def generate_generalized_integer(N, bias):79    """3.3 Generalized variable-length integers"""80    result = bytearray()81    j = 082    while 1:83        t = T(j, bias)84        if N < t:85            result.append(digits[N])86            return bytes(result)87        result.append(digits[t + ((N - t) % (36 - t))])88        N = (N - t) // (36 - t)89        j += 190 91def adapt(delta, first, numchars):92    if first:93        delta //= 70094    else:95        delta //= 296    delta += delta // numchars97    # ((base - tmin) * tmax) // 2 == 45598    divisions = 099    while delta > 455:100        delta = delta // 35 # base - tmin101        divisions += 36102    bias = divisions + (36 * delta // (delta + 38))103    return bias104 105 106def generate_integers(baselen, deltas):107    """3.4 Bias adaptation"""108    # Punycode parameters: initial bias = 72, damp = 700, skew = 38109    result = bytearray()110    bias = 72111    for points, delta in enumerate(deltas):112        s = generate_generalized_integer(delta, bias)113        result.extend(s)114        bias = adapt(delta, points==0, baselen+points+1)115    return bytes(result)116 117def punycode_encode(text):118    base, extended = segregate(text)119    deltas = insertion_unsort(text, extended)120    extended = generate_integers(len(base), deltas)121    if base:122        return base + b"-" + extended123    return extended124 125##################### Decoding #####################################126 127def decode_generalized_number(extended, extpos, bias, errors):128    """3.3 Generalized variable-length integers"""129    result = 0130    w = 1131    j = 0132    while 1:133        try:134            char = extended[extpos]135        except IndexError:136            if errors == "strict":137                raise UnicodeDecodeError("punycode", extended, extpos, extpos+1,138                                         "incomplete punycode string")139            return extpos + 1, None140        extpos += 1141        if 0x41 <= char <= 0x5A: # A-Z142            digit = char - 0x41143        elif 0x30 <= char <= 0x39:144            digit = char - 22 # 0x30-26145        elif errors == "strict":146            raise UnicodeDecodeError("punycode", extended, extpos-1, extpos,147                                     f"Invalid extended code point '{extended[extpos-1]}'")148        else:149            return extpos, None150        t = T(j, bias)151        result += digit * w152        if digit < t:153            return extpos, result154        w = w * (36 - t)155        j += 1156 157 158def insertion_sort(base, extended, errors):159    """3.2 Insertion sort coding"""160    # This function raises UnicodeDecodeError with position in the extended.161    # Caller should add the offset.162    char = 0x80163    pos = -1164    bias = 72165    extpos = 0166 167    while extpos < len(extended):168        newpos, delta = decode_generalized_number(extended, extpos,169                                                  bias, errors)170        if delta is None:171            # There was an error in decoding. We can't continue because172            # synchronization is lost.173            return base174        pos += delta+1175        char += pos // (len(base) + 1)176        if char > 0x10FFFF:177            if errors == "strict":178                raise UnicodeDecodeError(179                    "punycode", extended, pos-1, pos,180                    f"Invalid character U+{char:x}")181            char = ord('?')182        pos = pos % (len(base) + 1)183        base = base[:pos] + chr(char) + base[pos:]184        bias = adapt(delta, (extpos == 0), len(base))185        extpos = newpos186    return base187 188def punycode_decode(text, errors):189    if isinstance(text, str):190        text = text.encode("ascii")191    if isinstance(text, memoryview):192        text = bytes(text)193    pos = text.rfind(b"-")194    if pos == -1:195        base = ""196        extended = text.upper()197    else:198        try:199            base = str(text[:pos], "ascii", errors)200        except UnicodeDecodeError as exc:201            raise UnicodeDecodeError("ascii", text, exc.start, exc.end,202                                     exc.reason) from None203        extended = text[pos+1:].upper()204    try:205        return insertion_sort(base, extended, errors)206    except UnicodeDecodeError as exc:207        offset = pos + 1208        raise UnicodeDecodeError("punycode", text,209                                 offset+exc.start, offset+exc.end,210                                 exc.reason) from None211 212### Codec APIs213 214class Codec(codecs.Codec):215 216    def encode(self, input, errors='strict'):217        res = punycode_encode(input)218        return res, len(input)219 220    def decode(self, input, errors='strict'):221        if errors not in ('strict', 'replace', 'ignore'):222            raise UnicodeError(f"Unsupported error handling: {errors}")223        res = punycode_decode(input, errors)224        return res, len(input)225 226class IncrementalEncoder(codecs.IncrementalEncoder):227    def encode(self, input, final=False):228        return punycode_encode(input)229 230class IncrementalDecoder(codecs.IncrementalDecoder):231    def decode(self, input, final=False):232        if self.errors not in ('strict', 'replace', 'ignore'):233            raise UnicodeError(f"Unsupported error handling: {self.errors}")234        return punycode_decode(input, self.errors)235 236class StreamWriter(Codec,codecs.StreamWriter):237    pass238 239class StreamReader(Codec,codecs.StreamReader):240    pass241 242### encodings module API243 244def getregentry():245    return codecs.CodecInfo(246        name='punycode',247        encode=Codec().encode,248        decode=Codec().decode,249        incrementalencoder=IncrementalEncoder,250        incrementaldecoder=IncrementalDecoder,251        streamwriter=StreamWriter,252        streamreader=StreamReader,253    )254