codekingpro/portable-devtools

1"""2    pygments.cmdline3    ~~~~~~~~~~~~~~~~4 5    Command line interface.6 7    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11import os12import sys13import shutil14import argparse15from textwrap import dedent16 17from pygments import __version__, highlight18from pygments.util import ClassNotFound, OptionError, docstring_headline, \19    guess_decode, guess_decode_from_terminal, terminal_encoding, \20    UnclosingTextIOWrapper21from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \22    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename23from pygments.lexers.special import TextLexer24from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter25from pygments.formatters import get_all_formatters, get_formatter_by_name, \26    load_formatter_from_file, get_formatter_for_filename, find_formatter_class27from pygments.formatters.terminal import TerminalFormatter28from pygments.formatters.terminal256 import Terminal256Formatter, TerminalTrueColorFormatter29from pygments.filters import get_all_filters, find_filter_class30from pygments.styles import get_all_styles, get_style_by_name31 32 33def _parse_options(o_strs):34    opts = {}35    if not o_strs:36        return opts37    for o_str in o_strs:38        if not o_str.strip():39            continue40        o_args = o_str.split(',')41        for o_arg in o_args:42            o_arg = o_arg.strip()43            try:44                o_key, o_val = o_arg.split('=', 1)45                o_key = o_key.strip()46                o_val = o_val.strip()47            except ValueError:48                opts[o_arg] = True49            else:50                opts[o_key] = o_val51    return opts52 53 54def _parse_filters(f_strs):55    filters = []56    if not f_strs:57        return filters58    for f_str in f_strs:59        if ':' in f_str:60            fname, fopts = f_str.split(':', 1)61            filters.append((fname, _parse_options([fopts])))62        else:63            filters.append((f_str, {}))64    return filters65 66 67def _print_help(what, name):68    try:69        if what == 'lexer':70            cls = get_lexer_by_name(name)71            print(f"Help on the {cls.name} lexer:")72            print(dedent(cls.__doc__))73        elif what == 'formatter':74            cls = find_formatter_class(name)75            print(f"Help on the {cls.name} formatter:")76            print(dedent(cls.__doc__))77        elif what == 'filter':78            cls = find_filter_class(name)79            print(f"Help on the {name} filter:")80            print(dedent(cls.__doc__))81        return 082    except (AttributeError, ValueError):83        print(f"{what} not found!", file=sys.stderr)84        return 185 86 87def _print_list(what):88    if what == 'lexer':89        print()90        print("Lexers:")91        print("~~~~~~~")92 93        info = []94        for fullname, names, exts, _ in get_all_lexers():95            tup = (', '.join(names)+':', fullname,96                   exts and '(filenames ' + ', '.join(exts) + ')' or '')97            info.append(tup)98        info.sort()99        for i in info:100            print(('* {}\n    {} {}').format(*i))101 102    elif what == 'formatter':103        print()104        print("Formatters:")105        print("~~~~~~~~~~~")106 107        info = []108        for cls in get_all_formatters():109            doc = docstring_headline(cls)110            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and111                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')112            info.append(tup)113        info.sort()114        for i in info:115            print(('* {}\n    {} {}').format(*i))116 117    elif what == 'filter':118        print()119        print("Filters:")120        print("~~~~~~~~")121 122        for name in get_all_filters():123            cls = find_filter_class(name)124            print("* " + name + ':')125            print(f"    {docstring_headline(cls)}")126 127    elif what == 'style':128        print()129        print("Styles:")130        print("~~~~~~~")131 132        for name in get_all_styles():133            cls = get_style_by_name(name)134            print("* " + name + ':')135            print(f"    {docstring_headline(cls)}")136 137 138def _print_list_as_json(requested_items):139    import json140    result = {}141    if 'lexer' in requested_items:142        info = {}143        for fullname, names, filenames, mimetypes in get_all_lexers():144            info[fullname] = {145                'aliases': names,146                'filenames': filenames,147                'mimetypes': mimetypes148            }149        result['lexers'] = info150 151    if 'formatter' in requested_items:152        info = {}153        for cls in get_all_formatters():154            doc = docstring_headline(cls)155            info[cls.name] = {156                'aliases': cls.aliases,157                'filenames': cls.filenames,158                'doc': doc159            }160        result['formatters'] = info161 162    if 'filter' in requested_items:163        info = {}164        for name in get_all_filters():165            cls = find_filter_class(name)166            info[name] = {167                'doc': docstring_headline(cls)168            }169        result['filters'] = info170 171    if 'style' in requested_items:172        info = {}173        for name in get_all_styles():174            cls = get_style_by_name(name)175            info[name] = {176                'doc': docstring_headline(cls)177            }178        result['styles'] = info179 180    json.dump(result, sys.stdout)181 182def main_inner(parser, argns):183    if argns.help:184        parser.print_help()185        return 0186 187    if argns.V:188        print(f'Pygments version {__version__}, (c) 2006-present by Georg Brandl, Matthäus '189              'Chajdas and contributors.')190        return 0191 192    def is_only_option(opt):193        return not any(v for (k, v) in vars(argns).items() if k != opt)194 195    # handle ``pygmentize -L``196    if argns.L is not None:197        arg_set = set()198        for k, v in vars(argns).items():199            if v:200                arg_set.add(k)201 202        arg_set.discard('L')203        arg_set.discard('json')204 205        if arg_set:206            parser.print_help(sys.stderr)207            return 2208 209        # print version210        if not argns.json:211            main(['', '-V'])212        allowed_types = {'lexer', 'formatter', 'filter', 'style'}213        largs = [arg.rstrip('s') for arg in argns.L]214        if any(arg not in allowed_types for arg in largs):215            parser.print_help(sys.stderr)216            return 0217        if not largs:218            largs = allowed_types219        if not argns.json:220            for arg in largs:221                _print_list(arg)222        else:223            _print_list_as_json(largs)224        return 0225 226    # handle ``pygmentize -H``227    if argns.H:228        if not is_only_option('H'):229            parser.print_help(sys.stderr)230            return 2231        what, name = argns.H232        if what not in ('lexer', 'formatter', 'filter'):233            parser.print_help(sys.stderr)234            return 2235        return _print_help(what, name)236 237    # parse -O options238    parsed_opts = _parse_options(argns.O or [])239 240    # parse -P options241    for p_opt in argns.P or []:242        try:243            name, value = p_opt.split('=', 1)244        except ValueError:245            parsed_opts[p_opt] = True246        else:247            parsed_opts[name] = value248 249    # encodings250    inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))251    outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))252 253    # handle ``pygmentize -N``254    if argns.N:255        lexer = find_lexer_class_for_filename(argns.N)256        if lexer is None:257            lexer = TextLexer258 259        print(lexer.aliases[0])260        return 0261 262    # handle ``pygmentize -C``263    if argns.C:264        inp = sys.stdin.buffer.read()265        try:266            lexer = guess_lexer(inp, inencoding=inencoding)267        except ClassNotFound:268            lexer = TextLexer269 270        print(lexer.aliases[0])271        return 0272 273    # handle ``pygmentize -S``274    S_opt = argns.S275    a_opt = argns.a276    if S_opt is not None:277        f_opt = argns.f278        if not f_opt:279            parser.print_help(sys.stderr)280            return 2281        if argns.l or argns.INPUTFILE:282            parser.print_help(sys.stderr)283            return 2284 285        try:286            parsed_opts['style'] = S_opt287            fmter = get_formatter_by_name(f_opt, **parsed_opts)288        except ClassNotFound as err:289            print(err, file=sys.stderr)290            return 1291 292        print(fmter.get_style_defs(a_opt or ''))293        return 0294 295    # if no -S is given, -a is not allowed296    if argns.a is not None:297        parser.print_help(sys.stderr)298        return 2299 300    # parse -F options301    F_opts = _parse_filters(argns.F or [])302 303    # -x: allow custom (eXternal) lexers and formatters304    allow_custom_lexer_formatter = bool(argns.x)305 306    # select lexer307    lexer = None308 309    # given by name?310    lexername = argns.l311    if lexername:312        # custom lexer, located relative to user's cwd313        if allow_custom_lexer_formatter and '.py' in lexername:314            try:315                filename = None316                name = None317                if ':' in lexername:318                    filename, name = lexername.rsplit(':', 1)319 320                    if '.py' in name:321                        # This can happen on Windows: If the lexername is322                        # C:\lexer.py -- return to normal load path in that case323                        name = None324 325                if filename and name:326                    lexer = load_lexer_from_file(filename, name,327                                                 **parsed_opts)328                else:329                    lexer = load_lexer_from_file(lexername, **parsed_opts)330            except ClassNotFound as err:331                print('Error:', err, file=sys.stderr)332                return 1333        else:334            try:335                lexer = get_lexer_by_name(lexername, **parsed_opts)336            except (OptionError, ClassNotFound) as err:337                print('Error:', err, file=sys.stderr)338                return 1339 340    # read input code341    code = None342 343    if argns.INPUTFILE:344        if argns.s:345            print('Error: -s option not usable when input file specified',346                  file=sys.stderr)347            return 2348 349        infn = argns.INPUTFILE350        try:351            with open(infn, 'rb') as infp:352                code = infp.read()353        except Exception as err:354            print('Error: cannot read infile:', err, file=sys.stderr)355            return 1356        if not inencoding:357            code, inencoding = guess_decode(code)358 359        # do we have to guess the lexer?360        if not lexer:361            try:362                lexer = get_lexer_for_filename(infn, code, **parsed_opts)363            except ClassNotFound as err:364                if argns.g:365                    try:366                        lexer = guess_lexer(code, **parsed_opts)367                    except ClassNotFound:368                        lexer = TextLexer(**parsed_opts)369                else:370                    print('Error:', err, file=sys.stderr)371                    return 1372            except OptionError as err:373                print('Error:', err, file=sys.stderr)374                return 1375 376    elif not argns.s:  # treat stdin as full file (-s support is later)377        # read code from terminal, always in binary mode since we want to378        # decode ourselves and be tolerant with it379        code = sys.stdin.buffer.read()  # use .buffer to get a binary stream380        if not inencoding:381            code, inencoding = guess_decode_from_terminal(code, sys.stdin)382            # else the lexer will do the decoding383        if not lexer:384            try:385                lexer = guess_lexer(code, **parsed_opts)386            except ClassNotFound:387                lexer = TextLexer(**parsed_opts)388 389    else:  # -s option needs a lexer with -l390        if not lexer:391            print('Error: when using -s a lexer has to be selected with -l',392                  file=sys.stderr)393            return 2394 395    # process filters396    for fname, fopts in F_opts:397        try:398            lexer.add_filter(fname, **fopts)399        except ClassNotFound as err:400            print('Error:', err, file=sys.stderr)401            return 1402 403    # select formatter404    outfn = argns.o405    fmter = argns.f406    if fmter:407        # custom formatter, located relative to user's cwd408        if allow_custom_lexer_formatter and '.py' in fmter:409            try:410                filename = None411                name = None412                if ':' in fmter:413                    # Same logic as above for custom lexer414                    filename, name = fmter.rsplit(':', 1)415 416                    if '.py' in name:417                        name = None418 419                if filename and name:420                    fmter = load_formatter_from_file(filename, name,421                                                     **parsed_opts)422                else:423                    fmter = load_formatter_from_file(fmter, **parsed_opts)424            except ClassNotFound as err:425                print('Error:', err, file=sys.stderr)426                return 1427        else:428            try:429                fmter = get_formatter_by_name(fmter, **parsed_opts)430            except (OptionError, ClassNotFound) as err:431                print('Error:', err, file=sys.stderr)432                return 1433 434    if outfn:435        if not fmter:436            try:437                fmter = get_formatter_for_filename(outfn, **parsed_opts)438            except (OptionError, ClassNotFound) as err:439                print('Error:', err, file=sys.stderr)440                return 1441        try:442            outfile = open(outfn, 'wb')443        except Exception as err:444            print('Error: cannot open outfile:', err, file=sys.stderr)445            return 1446    else:447        if not fmter:448            if os.environ.get('COLORTERM','') in ('truecolor', '24bit'):449                fmter = TerminalTrueColorFormatter(**parsed_opts)450            elif '256' in os.environ.get('TERM', ''):451                fmter = Terminal256Formatter(**parsed_opts)452            else:453                fmter = TerminalFormatter(**parsed_opts)454        outfile = sys.stdout.buffer455 456    # determine output encoding if not explicitly selected457    if not outencoding:458        if outfn:459            # output file? use lexer encoding for now (can still be None)460            fmter.encoding = inencoding461        else:462            # else use terminal encoding463            fmter.encoding = terminal_encoding(sys.stdout)464 465    # provide coloring under Windows, if possible466    if not outfn and sys.platform in ('win32', 'cygwin') and \467       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover468        # unfortunately colorama doesn't support binary streams on Py3469        outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)470        fmter.encoding = None471        try:472            import colorama.initialise473        except ImportError:474            pass475        else:476            outfile = colorama.initialise.wrap_stream(477                outfile, convert=None, strip=None, autoreset=False, wrap=True)478 479    # When using the LaTeX formatter and the option `escapeinside` is480    # specified, we need a special lexer which collects escaped text481    # before running the chosen language lexer.482    escapeinside = parsed_opts.get('escapeinside', '')483    if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):484        left = escapeinside[0]485        right = escapeinside[1]486        lexer = LatexEmbeddedLexer(left, right, lexer)487 488    # ... and do it!489    if not argns.s:490        # process whole input as per normal...491        try:492            highlight(code, lexer, fmter, outfile)493        finally:494            if outfn:495                outfile.close()496        return 0497    else:498        # line by line processing of stdin (eg: for 'tail -f')...499        try:500            while 1:501                line = sys.stdin.buffer.readline()502                if not line:503                    break504                if not inencoding:505                    line = guess_decode_from_terminal(line, sys.stdin)[0]506                highlight(line, lexer, fmter, outfile)507                if hasattr(outfile, 'flush'):508                    outfile.flush()509            return 0510        except KeyboardInterrupt:  # pragma: no cover511            return 0512        finally:513            if outfn:514                outfile.close()515 516 517class HelpFormatter(argparse.HelpFormatter):518    def __init__(self, prog, indent_increment=2, max_help_position=16, width=None):519        if width is None:520            try:521                width = shutil.get_terminal_size().columns - 2522            except Exception:523                pass524        argparse.HelpFormatter.__init__(self, prog, indent_increment,525                                        max_help_position, width)526 527 528def main(args=sys.argv):529    """530    Main command line entry point.531    """532    desc = "Highlight an input file and write the result to an output file."533    parser = argparse.ArgumentParser(description=desc, add_help=False,534                                     formatter_class=HelpFormatter)535 536    operation = parser.add_argument_group('Main operation')537    lexersel = operation.add_mutually_exclusive_group()538    lexersel.add_argument(539        '-l', metavar='LEXER',540        help='Specify the lexer to use.  (Query names with -L.)  If not '541        'given and -g is not present, the lexer is guessed from the filename.')542    lexersel.add_argument(543        '-g', action='store_true',544        help='Guess the lexer from the file contents, or pass through '545        'as plain text if nothing can be guessed.')546    operation.add_argument(547        '-F', metavar='FILTER[:options]', action='append',548        help='Add a filter to the token stream.  (Query names with -L.) '549        'Filter options are given after a colon if necessary.')550    operation.add_argument(551        '-f', metavar='FORMATTER',552        help='Specify the formatter to use.  (Query names with -L.) '553        'If not given, the formatter is guessed from the output filename, '554        'and defaults to the terminal formatter if the output is to the '555        'terminal or an unknown file extension.')556    operation.add_argument(557        '-O', metavar='OPTION=value[,OPTION=value,...]', action='append',558        help='Give options to the lexer and formatter as a comma-separated '559        'list of key-value pairs. '560        'Example: `-O bg=light,python=cool`.')561    operation.add_argument(562        '-P', metavar='OPTION=value', action='append',563        help='Give a single option to the lexer and formatter - with this '564        'you can pass options whose value contains commas and equal signs. '565        'Example: `-P "heading=Pygments, the Python highlighter"`.')566    operation.add_argument(567        '-o', metavar='OUTPUTFILE',568        help='Where to write the output.  Defaults to standard output.')569 570    operation.add_argument(571        'INPUTFILE', nargs='?',572        help='Where to read the input.  Defaults to standard input.')573 574    flags = parser.add_argument_group('Operation flags')575    flags.add_argument(576        '-v', action='store_true',577        help='Print a detailed traceback on unhandled exceptions, which '578        'is useful for debugging and bug reports.')579    flags.add_argument(580        '-s', action='store_true',581        help='Process lines one at a time until EOF, rather than waiting to '582        'process the entire file.  This only works for stdin, only for lexers '583        'with no line-spanning constructs, and is intended for streaming '584        'input such as you get from `tail -f`. '585        'Example usage: `tail -f sql.log | pygmentize -s -l sql`.')586    flags.add_argument(587        '-x', action='store_true',588        help='Allow custom lexers and formatters to be loaded from a .py file '589        'relative to the current working directory. For example, '590        '`-l ./customlexer.py -x`. By default, this option expects a file '591        'with a class named CustomLexer or CustomFormatter; you can also '592        'specify your own class name with a colon (`-l ./lexer.py:MyLexer`). '593        'Users should be very careful not to use this option with untrusted '594        'files, because it will import and run them.')595    flags.add_argument('--json', help='Output as JSON. This can '596        'be only used in conjunction with -L.',597        default=False,598        action='store_true')599 600    special_modes_group = parser.add_argument_group(601        'Special modes - do not do any highlighting')602    special_modes = special_modes_group.add_mutually_exclusive_group()603    special_modes.add_argument(604        '-S', metavar='STYLE -f formatter',605        help='Print style definitions for STYLE for a formatter '606        'given with -f. The argument given by -a is formatter '607        'dependent.')608    special_modes.add_argument(609        '-L', nargs='*', metavar='WHAT',610        help='List lexers, formatters, styles or filters -- '611        'give additional arguments for the thing(s) you want to list '612        '(e.g. "styles"), or omit them to list everything.')613    special_modes.add_argument(614        '-N', metavar='FILENAME',615        help='Guess and print out a lexer name based solely on the given '616        'filename. Does not take input or highlight anything. If no specific '617        'lexer can be determined, "text" is printed.')618    special_modes.add_argument(619        '-C', action='store_true',620        help='Like -N, but print out a lexer name based solely on '621        'a given content from standard input.')622    special_modes.add_argument(623        '-H', action='store', nargs=2, metavar=('NAME', 'TYPE'),624        help='Print detailed help for the object <name> of type <type>, '625        'where <type> is one of "lexer", "formatter" or "filter".')626    special_modes.add_argument(627        '-V', action='store_true',628        help='Print the package version.')629    special_modes.add_argument(630        '-h', '--help', action='store_true',631        help='Print this help.')632    special_modes_group.add_argument(633        '-a', metavar='ARG',634        help='Formatter-specific additional argument for the -S (print '635        'style sheet) mode.')636 637    argns = parser.parse_args(args[1:])638 639    try:640        return main_inner(parser, argns)641    except BrokenPipeError:642        # someone closed our stdout, e.g. by quitting a pager.643        return 0644    except Exception:645        if argns.v:646            print(file=sys.stderr)647            print('*' * 65, file=sys.stderr)648            print('An unhandled exception occurred while highlighting.',649                  file=sys.stderr)650            print('Please report the whole traceback to the issue tracker at',651                  file=sys.stderr)652            print('<https://github.com/pygments/pygments/issues>.',653                  file=sys.stderr)654            print('*' * 65, file=sys.stderr)655            print(file=sys.stderr)656            raise657        import traceback658        info = traceback.format_exception(*sys.exc_info())659        msg = info[-1].strip()660        if len(info) >= 3:661            # extract relevant file and position info662            msg += '\n   (f{})'.format(info[-2].split('\n')[0].strip()[1:])663        print(file=sys.stderr)664        print('*** Error while highlighting:', file=sys.stderr)665        print(msg, file=sys.stderr)666        print('*** If this is a bug you want to report, please rerun with -v.',667              file=sys.stderr)668        return 1669