codekingpro/portable-devtools

1"""2    pygments.lexers3    ~~~~~~~~~~~~~~~4 5    Pygments lexers.6 7    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11import re12import sys13import types14import fnmatch15from os.path import basename16 17from pygments.lexers._mapping import LEXERS18from pygments.modeline import get_filetype_from_buffer19from pygments.plugin import find_plugin_lexers20from pygments.util import ClassNotFound, guess_decode21 22COMPAT = {23    'Python3Lexer': 'PythonLexer',24    'Python3TracebackLexer': 'PythonTracebackLexer',25    'LeanLexer': 'Lean3Lexer',26}27 28__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',29           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS) + list(COMPAT)30 31_lexer_cache = {}32_pattern_cache = {}33 34 35def _fn_matches(fn, glob):36    """Return whether the supplied file name fn matches pattern filename."""37    if glob not in _pattern_cache:38        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))39        return pattern.match(fn)40    return _pattern_cache[glob].match(fn)41 42 43def _load_lexers(module_name):44    """Load a lexer (and all others in the module too)."""45    mod = __import__(module_name, None, None, ['__all__'])46    for lexer_name in mod.__all__:47        cls = getattr(mod, lexer_name)48        _lexer_cache[cls.name] = cls49 50 51def get_all_lexers(plugins=True):52    """Return a generator of tuples in the form ``(name, aliases,53    filenames, mimetypes)`` of all know lexers.54 55    If *plugins* is true (the default), plugin lexers supplied by entrypoints56    are also returned.  Otherwise, only builtin ones are considered.57    """58    for item in LEXERS.values():59        yield item[1:]60    if plugins:61        for lexer in find_plugin_lexers():62            yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes63 64 65def find_lexer_class(name):66    """67    Return the `Lexer` subclass that with the *name* attribute as given by68    the *name* argument.69    """70    if name in _lexer_cache:71        return _lexer_cache[name]72    # lookup builtin lexers73    for module_name, lname, aliases, _, _ in LEXERS.values():74        if name == lname:75            _load_lexers(module_name)76            return _lexer_cache[name]77    # continue with lexers from setuptools entrypoints78    for cls in find_plugin_lexers():79        if cls.name == name:80            return cls81 82 83def find_lexer_class_by_name(_alias):84    """85    Return the `Lexer` subclass that has `alias` in its aliases list, without86    instantiating it.87 88    Like `get_lexer_by_name`, but does not instantiate the class.89 90    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is91    found.92 93    .. versionadded:: 2.294    """95    if not _alias:96        raise ClassNotFound(f'no lexer for alias {_alias!r} found')97    # lookup builtin lexers98    for module_name, name, aliases, _, _ in LEXERS.values():99        if _alias.lower() in aliases:100            if name not in _lexer_cache:101                _load_lexers(module_name)102            return _lexer_cache[name]103    # continue with lexers from setuptools entrypoints104    for cls in find_plugin_lexers():105        if _alias.lower() in cls.aliases:106            return cls107    raise ClassNotFound(f'no lexer for alias {_alias!r} found')108 109 110def get_lexer_by_name(_alias, **options):111    """112    Return an instance of a `Lexer` subclass that has `alias` in its113    aliases list. The lexer is given the `options` at its114    instantiation.115 116    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is117    found.118    """119    if not _alias:120        raise ClassNotFound(f'no lexer for alias {_alias!r} found')121 122    # lookup builtin lexers123    for module_name, name, aliases, _, _ in LEXERS.values():124        if _alias.lower() in aliases:125            if name not in _lexer_cache:126                _load_lexers(module_name)127            return _lexer_cache[name](**options)128    # continue with lexers from setuptools entrypoints129    for cls in find_plugin_lexers():130        if _alias.lower() in cls.aliases:131            return cls(**options)132    raise ClassNotFound(f'no lexer for alias {_alias!r} found')133 134 135def load_lexer_from_file(filename, lexername="CustomLexer", **options):136    """Load a lexer from a file.137 138    This method expects a file located relative to the current working139    directory, which contains a Lexer class. By default, it expects the140    Lexer to be name CustomLexer; you can specify your own class name141    as the second argument to this function.142 143    Users should be very careful with the input, because this method144    is equivalent to running eval on the input file.145 146    Raises ClassNotFound if there are any problems importing the Lexer.147 148    .. versionadded:: 2.2149    """150    try:151        # This empty dict will contain the namespace for the exec'd file152        custom_namespace = {}153        with open(filename, 'rb') as f:154            exec(f.read(), custom_namespace)155        # Retrieve the class `lexername` from that namespace156        if lexername not in custom_namespace:157            raise ClassNotFound(f'no valid {lexername} class found in {filename}')158        lexer_class = custom_namespace[lexername]159        # And finally instantiate it with the options160        return lexer_class(**options)161    except OSError as err:162        raise ClassNotFound(f'cannot read {filename}: {err}')163    except ClassNotFound:164        raise165    except Exception as err:166        raise ClassNotFound(f'error when loading custom lexer: {err}')167 168 169def find_lexer_class_for_filename(_fn, code=None):170    """Get a lexer for a filename.171 172    If multiple lexers match the filename pattern, use ``analyse_text()`` to173    figure out which one is more appropriate.174 175    Returns None if not found.176    """177    matches = []178    fn = basename(_fn)179    for modname, name, _, filenames, _ in LEXERS.values():180        for filename in filenames:181            if _fn_matches(fn, filename):182                if name not in _lexer_cache:183                    _load_lexers(modname)184                matches.append((_lexer_cache[name], filename))185    for cls in find_plugin_lexers():186        for filename in cls.filenames:187            if _fn_matches(fn, filename):188                matches.append((cls, filename))189 190    if isinstance(code, bytes):191        # decode it, since all analyse_text functions expect unicode192        code = guess_decode(code)193 194    def get_rating(info):195        cls, filename = info196        # explicit patterns get a bonus197        bonus = '*' not in filename and 0.5 or 0198        # The class _always_ defines analyse_text because it's included in199        # the Lexer class.  The default implementation returns None which200        # gets turned into 0.0.  Run scripts/detect_missing_analyse_text.py201        # to find lexers which need it overridden.202        if code:203            return cls.analyse_text(code) + bonus, cls.__name__204        return cls.priority + bonus, cls.__name__205 206    if matches:207        matches.sort(key=get_rating)208        # print "Possible lexers, after sort:", matches209        return matches[-1][0]210 211 212def get_lexer_for_filename(_fn, code=None, **options):213    """Get a lexer for a filename.214 215    Return a `Lexer` subclass instance that has a filename pattern216    matching `fn`. The lexer is given the `options` at its217    instantiation.218 219    Raise :exc:`pygments.util.ClassNotFound` if no lexer for that filename220    is found.221 222    If multiple lexers match the filename pattern, use their ``analyse_text()``223    methods to figure out which one is more appropriate.224    """225    res = find_lexer_class_for_filename(_fn, code)226    if not res:227        raise ClassNotFound(f'no lexer for filename {_fn!r} found')228    return res(**options)229 230 231def get_lexer_for_mimetype(_mime, **options):232    """233    Return a `Lexer` subclass instance that has `mime` in its mimetype234    list. The lexer is given the `options` at its instantiation.235 236    Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype237    is found.238    """239    for modname, name, _, _, mimetypes in LEXERS.values():240        if _mime in mimetypes:241            if name not in _lexer_cache:242                _load_lexers(modname)243            return _lexer_cache[name](**options)244    for cls in find_plugin_lexers():245        if _mime in cls.mimetypes:246            return cls(**options)247    raise ClassNotFound(f'no lexer for mimetype {_mime!r} found')248 249 250def _iter_lexerclasses(plugins=True):251    """Return an iterator over all lexer classes."""252    for key in sorted(LEXERS):253        module_name, name = LEXERS[key][:2]254        if name not in _lexer_cache:255            _load_lexers(module_name)256        yield _lexer_cache[name]257    if plugins:258        yield from find_plugin_lexers()259 260 261def guess_lexer_for_filename(_fn, _text, **options):262    """263    As :func:`guess_lexer()`, but only lexers which have a pattern in `filenames`264    or `alias_filenames` that matches `filename` are taken into consideration.265 266    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can267    handle the content.268    """269    fn = basename(_fn)270    primary = {}271    matching_lexers = set()272    for lexer in _iter_lexerclasses():273        for filename in lexer.filenames:274            if _fn_matches(fn, filename):275                matching_lexers.add(lexer)276                primary[lexer] = True277        for filename in lexer.alias_filenames:278            if _fn_matches(fn, filename):279                matching_lexers.add(lexer)280                primary[lexer] = False281    if not matching_lexers:282        raise ClassNotFound(f'no lexer for filename {fn!r} found')283    if len(matching_lexers) == 1:284        return matching_lexers.pop()(**options)285    result = []286    for lexer in matching_lexers:287        rv = lexer.analyse_text(_text)288        if rv == 1.0:289            return lexer(**options)290        result.append((rv, lexer))291 292    def type_sort(t):293        # sort by:294        # - analyse score295        # - is primary filename pattern?296        # - priority297        # - last resort: class name298        return (t[0], primary[t[1]], t[1].priority, t[1].__name__)299    result.sort(key=type_sort)300 301    return result[-1][1](**options)302 303 304def guess_lexer(_text, **options):305    """306    Return a `Lexer` subclass instance that's guessed from the text in307    `text`. For that, the :meth:`.analyse_text()` method of every known lexer308    class is called with the text as argument, and the lexer which returned the309    highest value will be instantiated and returned.310 311    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can312    handle the content.313    """314 315    if not isinstance(_text, str):316        inencoding = options.get('inencoding', options.get('encoding'))317        if inencoding:318            _text = _text.decode(inencoding or 'utf8')319        else:320            _text, _ = guess_decode(_text)321 322    # try to get a vim modeline first323    ft = get_filetype_from_buffer(_text)324 325    if ft is not None:326        try:327            return get_lexer_by_name(ft, **options)328        except ClassNotFound:329            pass330 331    best_lexer = [0.0, None]332    for lexer in _iter_lexerclasses():333        rv = lexer.analyse_text(_text)334        if rv == 1.0:335            return lexer(**options)336        if rv > best_lexer[0]:337            best_lexer[:] = (rv, lexer)338    if not best_lexer[0] or best_lexer[1] is None:339        raise ClassNotFound('no lexer matching the text found')340    return best_lexer[1](**options)341 342 343class _automodule(types.ModuleType):344    """Automatically import lexers."""345 346    def __getattr__(self, name):347        info = LEXERS.get(name)348        if info:349            _load_lexers(info[0])350            cls = _lexer_cache[info[1]]351            setattr(self, name, cls)352            return cls353        if name in COMPAT:354            return getattr(self, COMPAT[name])355        raise AttributeError(name)356 357 358oldmod = sys.modules[__name__]359newmod = _automodule(__name__)360newmod.__dict__.update(oldmod.__dict__)361sys.modules[__name__] = newmod362del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types363