codekingpro/portable-devtools

1"""2    pygments.token3    ~~~~~~~~~~~~~~4 5    Basic token types and the standard tokens.6 7    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11 12class _TokenType(tuple):13    parent = None14 15    def split(self):16        buf = []17        node = self18        while node is not None:19            buf.append(node)20            node = node.parent21        buf.reverse()22        return buf23 24    def __init__(self, *args):25        # no need to call super.__init__26        self.subtypes = set()27 28    def __contains__(self, val):29        return self is val or (30            type(val) is self.__class__ and31            val[:len(self)] == self32        )33 34    def __getattr__(self, val):35        if not val or not val[0].isupper():36            return tuple.__getattribute__(self, val)37        new = _TokenType(self + (val,))38        setattr(self, val, new)39        self.subtypes.add(new)40        new.parent = self41        return new42 43    def __repr__(self):44        return 'Token' + (self and '.' or '') + '.'.join(self)45 46    def __copy__(self):47        # These instances are supposed to be singletons48        return self49 50    def __deepcopy__(self, memo):51        # These instances are supposed to be singletons52        return self53 54 55Token = _TokenType()56 57# Special token types58Text = Token.Text59Whitespace = Text.Whitespace60Escape = Token.Escape61Error = Token.Error62# Text that doesn't belong to this lexer (e.g. HTML in PHP)63Other = Token.Other64 65# Common token types for source code66Keyword = Token.Keyword67Name = Token.Name68Literal = Token.Literal69String = Literal.String70Number = Literal.Number71Punctuation = Token.Punctuation72Operator = Token.Operator73Comment = Token.Comment74 75# Generic types for non-source code76Generic = Token.Generic77 78# String and some others are not direct children of Token.79# alias them:80Token.Token = Token81Token.String = String82Token.Number = Number83 84 85def is_token_subtype(ttype, other):86    """87    Return True if ``ttype`` is a subtype of ``other``.88 89    exists for backwards compatibility. use ``ttype in other`` now.90    """91    return ttype in other92 93 94def string_to_tokentype(s):95    """96    Convert a string into a token type::97 98        >>> string_to_token('String.Double')99        Token.Literal.String.Double100        >>> string_to_token('Token.Literal.Number')101        Token.Literal.Number102        >>> string_to_token('')103        Token104 105    Tokens that are already tokens are returned unchanged:106 107        >>> string_to_token(String)108        Token.Literal.String109    """110    if isinstance(s, _TokenType):111        return s112    if not s:113        return Token114    node = Token115    for item in s.split('.'):116        node = getattr(node, item)117    return node118 119 120# Map standard token types to short names, used in CSS class naming.121# If you add a new item, please be sure to run this file to perform122# a consistency check for duplicate values.123STANDARD_TYPES = {124    Token:                         '',125 126    Text:                          '',127    Whitespace:                    'w',128    Escape:                        'esc',129    Error:                         'err',130    Other:                         'x',131 132    Keyword:                       'k',133    Keyword.Constant:              'kc',134    Keyword.Declaration:           'kd',135    Keyword.Namespace:             'kn',136    Keyword.Pseudo:                'kp',137    Keyword.Reserved:              'kr',138    Keyword.Type:                  'kt',139 140    Name:                          'n',141    Name.Attribute:                'na',142    Name.Builtin:                  'nb',143    Name.Builtin.Pseudo:           'bp',144    Name.Class:                    'nc',145    Name.Constant:                 'no',146    Name.Decorator:                'nd',147    Name.Entity:                   'ni',148    Name.Exception:                'ne',149    Name.Function:                 'nf',150    Name.Function.Magic:           'fm',151    Name.Property:                 'py',152    Name.Label:                    'nl',153    Name.Namespace:                'nn',154    Name.Other:                    'nx',155    Name.Tag:                      'nt',156    Name.Variable:                 'nv',157    Name.Variable.Class:           'vc',158    Name.Variable.Global:          'vg',159    Name.Variable.Instance:        'vi',160    Name.Variable.Magic:           'vm',161 162    Literal:                       'l',163    Literal.Date:                  'ld',164 165    String:                        's',166    String.Affix:                  'sa',167    String.Backtick:               'sb',168    String.Char:                   'sc',169    String.Delimiter:              'dl',170    String.Doc:                    'sd',171    String.Double:                 's2',172    String.Escape:                 'se',173    String.Heredoc:                'sh',174    String.Interpol:               'si',175    String.Other:                  'sx',176    String.Regex:                  'sr',177    String.Single:                 's1',178    String.Symbol:                 'ss',179 180    Number:                        'm',181    Number.Bin:                    'mb',182    Number.Float:                  'mf',183    Number.Hex:                    'mh',184    Number.Integer:                'mi',185    Number.Integer.Long:           'il',186    Number.Oct:                    'mo',187 188    Operator:                      'o',189    Operator.Word:                 'ow',190 191    Punctuation:                   'p',192    Punctuation.Marker:            'pm',193 194    Comment:                       'c',195    Comment.Hashbang:              'ch',196    Comment.Multiline:             'cm',197    Comment.Preproc:               'cp',198    Comment.PreprocFile:           'cpf',199    Comment.Single:                'c1',200    Comment.Special:               'cs',201 202    Generic:                       'g',203    Generic.Deleted:               'gd',204    Generic.Emph:                  'ge',205    Generic.Error:                 'gr',206    Generic.Heading:               'gh',207    Generic.Inserted:              'gi',208    Generic.Output:                'go',209    Generic.Prompt:                'gp',210    Generic.Strong:                'gs',211    Generic.Subheading:            'gu',212    Generic.EmphStrong:            'ges',213    Generic.Traceback:             'gt',214}215