codekingpro/portable-devtools

1"""2    pygments.lexers.julia3    ~~~~~~~~~~~~~~~~~~~~~4 5    Lexers for the Julia language.6 7    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \12    words, include13from pygments.token import Text, Comment, Operator, Keyword, Name, String, \14    Number, Punctuation, Generic, Whitespace15from pygments.util import shebang_matches16from pygments.lexers._julia_builtins import OPERATORS_LIST, DOTTED_OPERATORS_LIST, \17    KEYWORD_LIST, BUILTIN_LIST, LITERAL_LIST18 19__all__ = ['JuliaLexer', 'JuliaConsoleLexer']20 21# see https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names22allowed_variable = \23    '(?:[a-zA-Z_\u00A1-\U0010ffff][a-zA-Z_0-9!\u00A1-\U0010ffff]*)'24# see https://github.com/JuliaLang/julia/blob/master/src/flisp/julia_opsuffs.h25operator_suffixes = r'[²³¹ʰʲʳʷʸˡˢˣᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵇᵈᵉᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₕₖₗₘₙₚₛₜⱼⱽ]*'26 27class JuliaLexer(RegexLexer):28    """29    For Julia source code.30    """31 32    name = 'Julia'33    url = 'https://julialang.org/'34    aliases = ['julia', 'jl']35    filenames = ['*.jl']36    mimetypes = ['text/x-julia', 'application/x-julia']37    version_added = '1.6'38 39    tokens = {40        'root': [41            (r'\n', Whitespace),42            (r'[^\S\n]+', Whitespace),43            (r'#=', Comment.Multiline, "blockcomment"),44            (r'#.*$', Comment),45            (r'[\[\](),;]', Punctuation),46 47            # symbols48            #   intercept range expressions first49            (r'(' + allowed_variable + r')(\s*)(:)(' + allowed_variable + ')',50                bygroups(Name, Whitespace, Operator, Name)),51            #   then match :name which does not follow closing brackets, digits, or the52            #   ::, <:, and :> operators53            (r'(?<![\]):<>\d.])(:' + allowed_variable + ')', String.Symbol),54 55            # type assertions - excludes expressions like ::typeof(sin) and ::avec[1]56            (r'(?<=::)(\s*)(' + allowed_variable + r')\b(?![(\[])',57             bygroups(Whitespace, Keyword.Type)),58            # type comparisons59            # - MyType <: A or MyType >: A60            ('(' + allowed_variable + r')(\s*)([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',61                bygroups(Keyword.Type, Whitespace, Operator, Whitespace, Keyword.Type)),62            # - <: B or >: B63            (r'([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',64                bygroups(Operator, Whitespace, Keyword.Type)),65            # - A <: or A >:66            (r'\b(' + allowed_variable + r')(\s*)([<>]:)',67                bygroups(Keyword.Type, Whitespace, Operator)),68 69            # operators70            # Suffixes aren't actually allowed on all operators, but we'll ignore that71            # since those cases are invalid Julia code.72            (words([*OPERATORS_LIST, *DOTTED_OPERATORS_LIST],73                   suffix=operator_suffixes), Operator),74            (words(['.' + o for o in DOTTED_OPERATORS_LIST],75                   suffix=operator_suffixes), Operator),76            (words(['...', '..']), Operator),77 78            # NOTE79            # Patterns below work only for definition sites and thus hardly reliable.80            #81            # functions82            # (r'(function)(\s+)(' + allowed_variable + ')',83            #  bygroups(Keyword, Text, Name.Function)),84 85            # chars86            (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|"87             r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),88 89            # try to match trailing transpose90            (r'(?<=[.\w)\]])(\'' + operator_suffixes + ')+', Operator),91 92            # raw strings93            (r'(raw)(""")', bygroups(String.Affix, String), 'tqrawstring'),94            (r'(raw)(")', bygroups(String.Affix, String), 'rawstring'),95            # regular expressions96            (r'(r)(""")', bygroups(String.Affix, String.Regex), 'tqregex'),97            (r'(r)(")', bygroups(String.Affix, String.Regex), 'regex'),98            # other strings99            (r'(' + allowed_variable + ')?(""")',100             bygroups(String.Affix, String), 'tqstring'),101            (r'(' + allowed_variable + ')?(")',102             bygroups(String.Affix, String), 'string'),103 104            # backticks105            (r'(' + allowed_variable + ')?(```)',106             bygroups(String.Affix, String.Backtick), 'tqcommand'),107            (r'(' + allowed_variable + ')?(`)',108             bygroups(String.Affix, String.Backtick), 'command'),109 110            # type names111            # - names that begin a curly expression112            ('(' + allowed_variable + r')(\{)',113                bygroups(Keyword.Type, Punctuation), 'curly'),114            # - names as part of bare 'where'115            (r'(where)(\s+)(' + allowed_variable + ')',116                bygroups(Keyword, Whitespace, Keyword.Type)),117            # - curly expressions in general118            (r'(\{)', Punctuation, 'curly'),119            # - names as part of type declaration120            (r'(abstract|primitive)([ \t]+)(type\b)([\s()]+)(' +121                allowed_variable + r')',122                bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),123            (r'(mutable(?=[ \t]))?([ \t]+)?(struct\b)([\s()]+)(' +124                allowed_variable + r')',125                bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),126 127            # macros128            (r'@' + allowed_variable, Name.Decorator),129            (words([*OPERATORS_LIST, '..', '.', *DOTTED_OPERATORS_LIST],130                prefix='@', suffix=operator_suffixes), Name.Decorator),131 132            # keywords133            (words(KEYWORD_LIST, suffix=r'\b'), Keyword),134            # builtin types135            (words(BUILTIN_LIST, suffix=r'\b'), Keyword.Type),136            # builtin literals137            (words(LITERAL_LIST, suffix=r'\b'), Name.Builtin),138 139            # names140            (allowed_variable, Name),141 142            # numbers143            (r'(\d+((_\d+)+)?\.(?!\.)(\d+((_\d+)+)?)?|\.\d+((_\d+)+)?)([eEf][+-]?[0-9]+)?', Number.Float),144            (r'\d+((_\d+)+)?[eEf][+-]?[0-9]+', Number.Float),145            (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?(\.([a-fA-F0-9]+((_[a-fA-F0-9]+)+)?)?)?p[+-]?\d+', Number.Float),146            (r'0b[01]+((_[01]+)+)?', Number.Bin),147            (r'0o[0-7]+((_[0-7]+)+)?', Number.Oct),148            (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?', Number.Hex),149            (r'\d+((_\d+)+)?', Number.Integer),150 151            # single dot operator matched last to permit e.g. ".1" as a float152            (words(['.']), Operator),153        ],154 155        "blockcomment": [156            (r'[^=#]', Comment.Multiline),157            (r'#=', Comment.Multiline, '#push'),158            (r'=#', Comment.Multiline, '#pop'),159            (r'[=#]', Comment.Multiline),160        ],161 162        'curly': [163            (r'\{', Punctuation, '#push'),164            (r'\}', Punctuation, '#pop'),165            (allowed_variable, Keyword.Type),166            include('root'),167        ],168 169        'tqrawstring': [170            (r'"""', String, '#pop'),171            (r'([^"]|"[^"][^"])+', String),172        ],173        'rawstring': [174            (r'"', String, '#pop'),175            (r'\\"', String.Escape),176            (r'([^"\\]|\\[^"])+', String),177        ],178 179        # Interpolation is defined as "$" followed by the shortest full180        # expression, which is something we can't parse.  Include the most181        # common cases here: $word, and $(paren'd expr).182        'interp': [183            (r'\$' + allowed_variable, String.Interpol),184            (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),185        ],186        'in-intp': [187            (r'\(', Punctuation, '#push'),188            (r'\)', Punctuation, '#pop'),189            include('root'),190        ],191 192        'string': [193            (r'(")(' + allowed_variable + r'|\d+)?',194             bygroups(String, String.Affix), '#pop'),195            # FIXME: This escape pattern is not perfect.196            (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),197            include('interp'),198            # @printf and @sprintf formats199            (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]',200             String.Interpol),201            (r'[^"$%\\]+', String),202            (r'.', String),203        ],204        'tqstring': [205            (r'(""")(' + allowed_variable + r'|\d+)?',206             bygroups(String, String.Affix), '#pop'),207            (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),208            include('interp'),209            (r'[^"$%\\]+', String),210            (r'.', String),211        ],212 213        'regex': [214            (r'(")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),215            (r'\\"', String.Regex),216            (r'[^\\"]+', String.Regex),217        ],218 219        'tqregex': [220            (r'(""")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),221            (r'[^"]+', String.Regex),222        ],223 224        'command': [225            (r'(`)(' + allowed_variable + r'|\d+)?',226             bygroups(String.Backtick, String.Affix), '#pop'),227            (r'\\[`$]', String.Escape),228            include('interp'),229            (r'[^\\`$]+', String.Backtick),230            (r'.', String.Backtick),231        ],232        'tqcommand': [233            (r'(```)(' + allowed_variable + r'|\d+)?',234             bygroups(String.Backtick, String.Affix), '#pop'),235            (r'\\\$', String.Escape),236            include('interp'),237            (r'[^\\`$]+', String.Backtick),238            (r'.', String.Backtick),239        ],240    }241 242    def analyse_text(text):243        return shebang_matches(text, r'julia')244 245 246class JuliaConsoleLexer(Lexer):247    """248    For Julia console sessions. Modeled after MatlabSessionLexer.249    """250    name = 'Julia console'251    aliases = ['jlcon', 'julia-repl']252    url = 'https://julialang.org/'253    version_added = '1.6'254    _example = "jlcon/console"255 256    def get_tokens_unprocessed(self, text):257        jllexer = JuliaLexer(**self.options)258        start = 0259        curcode = ''260        insertions = []261        output = False262        error = False263 264        for line in text.splitlines(keepends=True):265            if line.startswith('julia>'):266                insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])]))267                curcode += line[6:]268                output = False269                error = False270            elif line.startswith('help?>') or line.startswith('shell>'):271                yield start, Generic.Prompt, line[:6]272                yield start + 6, Text, line[6:]273                output = False274                error = False275            elif line.startswith('      ') and not output:276                insertions.append((len(curcode), [(0, Whitespace, line[:6])]))277                curcode += line[6:]278            else:279                if curcode:280                    yield from do_insertions(281                        insertions, jllexer.get_tokens_unprocessed(curcode))282                    curcode = ''283                    insertions = []284                if line.startswith('ERROR: ') or error:285                    yield start, Generic.Error, line286                    error = True287                else:288                    yield start, Generic.Output, line289                output = True290            start += len(line)291 292        if curcode:293            yield from do_insertions(294                insertions, jllexer.get_tokens_unprocessed(curcode))295