codekingpro/portable-devtools

1from . import idnadata2import bisect3import unicodedata4import re5from typing import Union, Optional6from .intranges import intranges_contain7 8_virama_combining_class = 99_alabel_prefix = b'xn--'10_unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')11 12class IDNAError(UnicodeError):13    """ Base exception for all IDNA-encoding related problems """14    pass15 16 17class IDNABidiError(IDNAError):18    """ Exception when bidirectional requirements are not satisfied """19    pass20 21 22class InvalidCodepoint(IDNAError):23    """ Exception when a disallowed or unallocated codepoint is used """24    pass25 26 27class InvalidCodepointContext(IDNAError):28    """ Exception when the codepoint is not valid in the context it is used """29    pass30 31 32def _combining_class(cp: int) -> int:33    v = unicodedata.combining(chr(cp))34    if v == 0:35        if not unicodedata.name(chr(cp)):36            raise ValueError('Unknown character in unicodedata')37    return v38 39def _is_script(cp: str, script: str) -> bool:40    return intranges_contain(ord(cp), idnadata.scripts[script])41 42def _punycode(s: str) -> bytes:43    return s.encode('punycode')44 45def _unot(s: int) -> str:46    return 'U+{:04X}'.format(s)47 48 49def valid_label_length(label: Union[bytes, str]) -> bool:50    if len(label) > 63:51        return False52    return True53 54 55def valid_string_length(label: Union[bytes, str], trailing_dot: bool) -> bool:56    if len(label) > (254 if trailing_dot else 253):57        return False58    return True59 60 61def check_bidi(label: str, check_ltr: bool = False) -> bool:62    # Bidi rules should only be applied if string contains RTL characters63    bidi_label = False64    for (idx, cp) in enumerate(label, 1):65        direction = unicodedata.bidirectional(cp)66        if direction == '':67            # String likely comes from a newer version of Unicode68            raise IDNABidiError('Unknown directionality in label {} at position {}'.format(repr(label), idx))69        if direction in ['R', 'AL', 'AN']:70            bidi_label = True71    if not bidi_label and not check_ltr:72        return True73 74    # Bidi rule 175    direction = unicodedata.bidirectional(label[0])76    if direction in ['R', 'AL']:77        rtl = True78    elif direction == 'L':79        rtl = False80    else:81        raise IDNABidiError('First codepoint in label {} must be directionality L, R or AL'.format(repr(label)))82 83    valid_ending = False84    number_type = None  # type: Optional[str]85    for (idx, cp) in enumerate(label, 1):86        direction = unicodedata.bidirectional(cp)87 88        if rtl:89            # Bidi rule 290            if not direction in ['R', 'AL', 'AN', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:91                raise IDNABidiError('Invalid direction for codepoint at position {} in a right-to-left label'.format(idx))92            # Bidi rule 393            if direction in ['R', 'AL', 'EN', 'AN']:94                valid_ending = True95            elif direction != 'NSM':96                valid_ending = False97            # Bidi rule 498            if direction in ['AN', 'EN']:99                if not number_type:100                    number_type = direction101                else:102                    if number_type != direction:103                        raise IDNABidiError('Can not mix numeral types in a right-to-left label')104        else:105            # Bidi rule 5106            if not direction in ['L', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:107                raise IDNABidiError('Invalid direction for codepoint at position {} in a left-to-right label'.format(idx))108            # Bidi rule 6109            if direction in ['L', 'EN']:110                valid_ending = True111            elif direction != 'NSM':112                valid_ending = False113 114    if not valid_ending:115        raise IDNABidiError('Label ends with illegal codepoint directionality')116 117    return True118 119 120def check_initial_combiner(label: str) -> bool:121    if unicodedata.category(label[0])[0] == 'M':122        raise IDNAError('Label begins with an illegal combining character')123    return True124 125 126def check_hyphen_ok(label: str) -> bool:127    if label[2:4] == '--':128        raise IDNAError('Label has disallowed hyphens in 3rd and 4th position')129    if label[0] == '-' or label[-1] == '-':130        raise IDNAError('Label must not start or end with a hyphen')131    return True132 133 134def check_nfc(label: str) -> None:135    if unicodedata.normalize('NFC', label) != label:136        raise IDNAError('Label must be in Normalization Form C')137 138 139def valid_contextj(label: str, pos: int) -> bool:140    cp_value = ord(label[pos])141 142    if cp_value == 0x200c:143 144        if pos > 0:145            if _combining_class(ord(label[pos - 1])) == _virama_combining_class:146                return True147 148        ok = False149        for i in range(pos-1, -1, -1):150            joining_type = idnadata.joining_types.get(ord(label[i]))151            if joining_type == ord('T'):152                continue153            elif joining_type in [ord('L'), ord('D')]:154                ok = True155                break156            else:157                break158 159        if not ok:160            return False161 162        ok = False163        for i in range(pos+1, len(label)):164            joining_type = idnadata.joining_types.get(ord(label[i]))165            if joining_type == ord('T'):166                continue167            elif joining_type in [ord('R'), ord('D')]:168                ok = True169                break170            else:171                break172        return ok173 174    if cp_value == 0x200d:175 176        if pos > 0:177            if _combining_class(ord(label[pos - 1])) == _virama_combining_class:178                return True179        return False180 181    else:182 183        return False184 185 186def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:187    cp_value = ord(label[pos])188 189    if cp_value == 0x00b7:190        if 0 < pos < len(label)-1:191            if ord(label[pos - 1]) == 0x006c and ord(label[pos + 1]) == 0x006c:192                return True193        return False194 195    elif cp_value == 0x0375:196        if pos < len(label)-1 and len(label) > 1:197            return _is_script(label[pos + 1], 'Greek')198        return False199 200    elif cp_value == 0x05f3 or cp_value == 0x05f4:201        if pos > 0:202            return _is_script(label[pos - 1], 'Hebrew')203        return False204 205    elif cp_value == 0x30fb:206        for cp in label:207            if cp == '\u30fb':208                continue209            if _is_script(cp, 'Hiragana') or _is_script(cp, 'Katakana') or _is_script(cp, 'Han'):210                return True211        return False212 213    elif 0x660 <= cp_value <= 0x669:214        for cp in label:215            if 0x6f0 <= ord(cp) <= 0x06f9:216                return False217        return True218 219    elif 0x6f0 <= cp_value <= 0x6f9:220        for cp in label:221            if 0x660 <= ord(cp) <= 0x0669:222                return False223        return True224 225    return False226 227 228def check_label(label: Union[str, bytes, bytearray]) -> None:229    if isinstance(label, (bytes, bytearray)):230        label = label.decode('utf-8')231    if len(label) == 0:232        raise IDNAError('Empty Label')233 234    check_nfc(label)235    check_hyphen_ok(label)236    check_initial_combiner(label)237 238    for (pos, cp) in enumerate(label):239        cp_value = ord(cp)240        if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']):241            continue242        elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']):243            if not valid_contextj(label, pos):244                raise InvalidCodepointContext('Joiner {} not allowed at position {} in {}'.format(245                    _unot(cp_value), pos+1, repr(label)))246        elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']):247            if not valid_contexto(label, pos):248                raise InvalidCodepointContext('Codepoint {} not allowed at position {} in {}'.format(_unot(cp_value), pos+1, repr(label)))249        else:250            raise InvalidCodepoint('Codepoint {} at position {} of {} not allowed'.format(_unot(cp_value), pos+1, repr(label)))251 252    check_bidi(label)253 254 255def alabel(label: str) -> bytes:256    try:257        label_bytes = label.encode('ascii')258        ulabel(label_bytes)259        if not valid_label_length(label_bytes):260            raise IDNAError('Label too long')261        return label_bytes262    except UnicodeEncodeError:263        pass264 265    check_label(label)266    label_bytes = _alabel_prefix + _punycode(label)267 268    if not valid_label_length(label_bytes):269        raise IDNAError('Label too long')270 271    return label_bytes272 273 274def ulabel(label: Union[str, bytes, bytearray]) -> str:275    if not isinstance(label, (bytes, bytearray)):276        try:277            label_bytes = label.encode('ascii')278        except UnicodeEncodeError:279            check_label(label)280            return label281    else:282        label_bytes = label283 284    label_bytes = label_bytes.lower()285    if label_bytes.startswith(_alabel_prefix):286        label_bytes = label_bytes[len(_alabel_prefix):]287        if not label_bytes:288            raise IDNAError('Malformed A-label, no Punycode eligible content found')289        if label_bytes.decode('ascii')[-1] == '-':290            raise IDNAError('A-label must not end with a hyphen')291    else:292        check_label(label_bytes)293        return label_bytes.decode('ascii')294 295    try:296        label = label_bytes.decode('punycode')297    except UnicodeError:298        raise IDNAError('Invalid A-label')299    check_label(label)300    return label301 302 303def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:304    """Re-map the characters in the string according to UTS46 processing."""305    from .uts46data import uts46data306    output = ''307 308    for pos, char in enumerate(domain):309        code_point = ord(char)310        try:311            uts46row = uts46data[code_point if code_point < 256 else312                bisect.bisect_left(uts46data, (code_point, 'Z')) - 1]313            status = uts46row[1]314            replacement = None  # type: Optional[str]315            if len(uts46row) == 3:316                replacement = uts46row[2]317            if (status == 'V' or318                    (status == 'D' and not transitional) or319                    (status == '3' and not std3_rules and replacement is None)):320                output += char321            elif replacement is not None and (status == 'M' or322                    (status == '3' and not std3_rules) or323                    (status == 'D' and transitional)):324                output += replacement325            elif status != 'I':326                raise IndexError()327        except IndexError:328            raise InvalidCodepoint(329                'Codepoint {} not allowed at position {} in {}'.format(330                _unot(code_point), pos + 1, repr(domain)))331 332    return unicodedata.normalize('NFC', output)333 334 335def encode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False, transitional: bool = False) -> bytes:336    if not isinstance(s, str):337        try:338            s = str(s, 'ascii')339        except UnicodeDecodeError:340            raise IDNAError('should pass a unicode string to the function rather than a byte string.')341    if uts46:342        s = uts46_remap(s, std3_rules, transitional)343    trailing_dot = False344    result = []345    if strict:346        labels = s.split('.')347    else:348        labels = _unicode_dots_re.split(s)349    if not labels or labels == ['']:350        raise IDNAError('Empty domain')351    if labels[-1] == '':352        del labels[-1]353        trailing_dot = True354    for label in labels:355        s = alabel(label)356        if s:357            result.append(s)358        else:359            raise IDNAError('Empty label')360    if trailing_dot:361        result.append(b'')362    s = b'.'.join(result)363    if not valid_string_length(s, trailing_dot):364        raise IDNAError('Domain too long')365    return s366 367 368def decode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False) -> str:369    try:370        if not isinstance(s, str):371            s = str(s, 'ascii')372    except UnicodeDecodeError:373        raise IDNAError('Invalid ASCII in A-label')374    if uts46:375        s = uts46_remap(s, std3_rules, False)376    trailing_dot = False377    result = []378    if not strict:379        labels = _unicode_dots_re.split(s)380    else:381        labels = s.split('.')382    if not labels or labels == ['']:383        raise IDNAError('Empty domain')384    if not labels[-1]:385        del labels[-1]386        trailing_dot = True387    for label in labels:388        s = ulabel(label)389        if s:390            result.append(s)391        else:392            raise IDNAError('Empty label')393    if trailing_dot:394        result.append('')395    return '.'.join(result)396