codekingpro/portable-devtools

1"""2The `ast` module helps Python applications to process trees of the Python3abstract syntax grammar.  The abstract syntax itself might change with4each Python release; this module helps to find out programmatically what5the current grammar looks like and allows modifications of it.6 7An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as8a flag to the `compile()` builtin function or by using the `parse()`9function from this module.  The result will be a tree of objects whose10classes all inherit from `ast.AST`.11 12A modified abstract syntax tree can be compiled into a Python code object13using the built-in `compile()` function.14 15Additionally various helper functions are provided that make working with16the trees simpler.  The main intention of the helper functions and this17module in general is to provide an easy to use interface for libraries18that work tightly with the python syntax (template engines for example).19 20:copyright: Copyright 2008 by Armin Ronacher.21:license: Python License.22"""23from _ast import *24 25 26def parse(source, filename='<unknown>', mode='exec', *,27          type_comments=False, feature_version=None, optimize=-1):28    """29    Parse the source into an AST node.30    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).31    Pass type_comments=True to get back type comments where the syntax allows.32    """33    flags = PyCF_ONLY_AST34    if optimize > 0:35        flags |= PyCF_OPTIMIZED_AST36    if type_comments:37        flags |= PyCF_TYPE_COMMENTS38    if feature_version is None:39        feature_version = -140    elif isinstance(feature_version, tuple):41        major, minor = feature_version  # Should be a 2-tuple.42        if major != 3:43            raise ValueError(f"Unsupported major version: {major}")44        feature_version = minor45    # Else it should be an int giving the minor version for 3.x.46    return compile(source, filename, mode, flags,47                   _feature_version=feature_version, optimize=optimize)48 49 50def literal_eval(node_or_string):51    """52    Evaluate an expression node or a string containing only a Python53    expression.  The string or node provided may only consist of the following54    Python literal structures: strings, bytes, numbers, tuples, lists, dicts,55    sets, booleans, and None.56 57    Caution: A complex expression can overflow the C stack and cause a crash.58    """59    if isinstance(node_or_string, str):60        node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval')61    if isinstance(node_or_string, Expression):62        node_or_string = node_or_string.body63    def _raise_malformed_node(node):64        msg = "malformed node or string"65        if lno := getattr(node, 'lineno', None):66            msg += f' on line {lno}'67        raise ValueError(msg + f': {node!r}')68    def _convert_num(node):69        if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):70            _raise_malformed_node(node)71        return node.value72    def _convert_signed_num(node):73        if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):74            operand = _convert_num(node.operand)75            if isinstance(node.op, UAdd):76                return + operand77            else:78                return - operand79        return _convert_num(node)80    def _convert(node):81        if isinstance(node, Constant):82            return node.value83        elif isinstance(node, Tuple):84            return tuple(map(_convert, node.elts))85        elif isinstance(node, List):86            return list(map(_convert, node.elts))87        elif isinstance(node, Set):88            return set(map(_convert, node.elts))89        elif (isinstance(node, Call) and isinstance(node.func, Name) and90              node.func.id == 'set' and node.args == node.keywords == []):91            return set()92        elif isinstance(node, Dict):93            if len(node.keys) != len(node.values):94                _raise_malformed_node(node)95            return dict(zip(map(_convert, node.keys),96                            map(_convert, node.values)))97        elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):98            left = _convert_signed_num(node.left)99            right = _convert_num(node.right)100            if isinstance(left, (int, float)) and isinstance(right, complex):101                if isinstance(node.op, Add):102                    return left + right103                else:104                    return left - right105        return _convert_signed_num(node)106    return _convert(node_or_string)107 108 109def dump(110    node, annotate_fields=True, include_attributes=False,111    *,112    indent=None, show_empty=False,113):114    """115    Return a formatted dump of the tree in node.  This is mainly useful for116    debugging purposes.  If annotate_fields is true (by default),117    the returned string will show the names and the values for fields.118    If annotate_fields is false, the result string will be more compact by119    omitting unambiguous field names.  Attributes such as line120    numbers and column offsets are not dumped by default.  If this is wanted,121    include_attributes can be set to true.  If indent is a non-negative122    integer or string, then the tree will be pretty-printed with that indent123    level. None (the default) selects the single line representation.124    If show_empty is False, then empty lists and fields that are None125    will be omitted from the output for better readability.126    """127    def _format(node, level=0):128        if indent is not None:129            level += 1130            prefix = '\n' + indent * level131            sep = ',\n' + indent * level132        else:133            prefix = ''134            sep = ', '135        if isinstance(node, AST):136            cls = type(node)137            args = []138            args_buffer = []139            allsimple = True140            keywords = annotate_fields141            for name in node._fields:142                try:143                    value = getattr(node, name)144                except AttributeError:145                    keywords = True146                    continue147                if value is None and getattr(cls, name, ...) is None:148                    keywords = True149                    continue150                if not show_empty:151                    if value == []:152                        field_type = cls._field_types.get(name, object)153                        if getattr(field_type, '__origin__', ...) is list:154                            if not keywords:155                                args_buffer.append(repr(value))156                            continue157                    if not keywords:158                        args.extend(args_buffer)159                        args_buffer = []160                value, simple = _format(value, level)161                allsimple = allsimple and simple162                if keywords:163                    args.append('%s=%s' % (name, value))164                else:165                    args.append(value)166            if include_attributes and node._attributes:167                for name in node._attributes:168                    try:169                        value = getattr(node, name)170                    except AttributeError:171                        continue172                    if value is None and getattr(cls, name, ...) is None:173                        continue174                    value, simple = _format(value, level)175                    allsimple = allsimple and simple176                    args.append('%s=%s' % (name, value))177            if allsimple and len(args) <= 3:178                return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args179            return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False180        elif isinstance(node, list):181            if not node:182                return '[]', True183            return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False184        return repr(node), True185 186    if not isinstance(node, AST):187        raise TypeError('expected AST, got %r' % node.__class__.__name__)188    if indent is not None and not isinstance(indent, str):189        indent = ' ' * indent190    return _format(node)[0]191 192 193def copy_location(new_node, old_node):194    """195    Copy source location (`lineno`, `col_offset`, `end_lineno`, and `end_col_offset`196    attributes) from *old_node* to *new_node* if possible, and return *new_node*.197    """198    for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset':199        if attr in old_node._attributes and attr in new_node._attributes:200            value = getattr(old_node, attr, None)201            # end_lineno and end_col_offset are optional attributes, and they202            # should be copied whether the value is None or not.203            if value is not None or (204                hasattr(old_node, attr) and attr.startswith("end_")205            ):206                setattr(new_node, attr, value)207    return new_node208 209 210def fix_missing_locations(node):211    """212    When you compile a node tree with compile(), the compiler expects lineno and213    col_offset attributes for every node that supports them.  This is rather214    tedious to fill in for generated nodes, so this helper adds these attributes215    recursively where not already set, by setting them to the values of the216    parent node.  It works recursively starting at *node*.217    """218    def _fix(node, lineno, col_offset, end_lineno, end_col_offset):219        if 'lineno' in node._attributes:220            if not hasattr(node, 'lineno'):221                node.lineno = lineno222            else:223                lineno = node.lineno224        if 'end_lineno' in node._attributes:225            if getattr(node, 'end_lineno', None) is None:226                node.end_lineno = end_lineno227            else:228                end_lineno = node.end_lineno229        if 'col_offset' in node._attributes:230            if not hasattr(node, 'col_offset'):231                node.col_offset = col_offset232            else:233                col_offset = node.col_offset234        if 'end_col_offset' in node._attributes:235            if getattr(node, 'end_col_offset', None) is None:236                node.end_col_offset = end_col_offset237            else:238                end_col_offset = node.end_col_offset239        for child in iter_child_nodes(node):240            _fix(child, lineno, col_offset, end_lineno, end_col_offset)241    _fix(node, 1, 0, 1, 0)242    return node243 244 245def increment_lineno(node, n=1):246    """247    Increment the line number and end line number of each node in the tree248    starting at *node* by *n*. This is useful to "move code" to a different249    location in a file.250    """251    for child in walk(node):252        # TypeIgnore is a special case where lineno is not an attribute253        # but rather a field of the node itself.254        if isinstance(child, TypeIgnore):255            child.lineno = getattr(child, 'lineno', 0) + n256            continue257 258        if 'lineno' in child._attributes:259            child.lineno = getattr(child, 'lineno', 0) + n260        if (261            "end_lineno" in child._attributes262            and (end_lineno := getattr(child, "end_lineno", 0)) is not None263        ):264            child.end_lineno = end_lineno + n265    return node266 267 268def iter_fields(node):269    """270    Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``271    that is present on *node*.272    """273    for field in node._fields:274        try:275            yield field, getattr(node, field)276        except AttributeError:277            pass278 279 280def iter_child_nodes(node):281    """282    Yield all direct child nodes of *node*, that is, all fields that are nodes283    and all items of fields that are lists of nodes.284    """285    for name, field in iter_fields(node):286        if isinstance(field, AST):287            yield field288        elif isinstance(field, list):289            for item in field:290                if isinstance(item, AST):291                    yield item292 293 294def get_docstring(node, clean=True):295    """296    Return the docstring for the given node or None if no docstring can297    be found.  If the node provided does not have docstrings a TypeError298    will be raised.299 300    If *clean* is `True`, all tabs are expanded to spaces and any whitespace301    that can be uniformly removed from the second line onwards is removed.302    """303    if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):304        raise TypeError("%r can't have docstrings" % node.__class__.__name__)305    if not(node.body and isinstance(node.body[0], Expr)):306        return None307    node = node.body[0].value308    if isinstance(node, Constant) and isinstance(node.value, str):309        text = node.value310    else:311        return None312    if clean:313        import inspect314        text = inspect.cleandoc(text)315    return text316 317 318_line_pattern = None319def _splitlines_no_ff(source, maxlines=None):320    """Split a string into lines ignoring form feed and other chars.321 322    This mimics how the Python parser splits source code.323    """324    global _line_pattern325    if _line_pattern is None:326        # lazily computed to speedup import time of `ast`327        import re328        _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))")329 330    lines = []331    for lineno, match in enumerate(_line_pattern.finditer(source), 1):332        if maxlines is not None and lineno > maxlines:333            break334        lines.append(match[0])335    return lines336 337 338def _pad_whitespace(source):339    r"""Replace all chars except '\f\t' in a line with spaces."""340    result = ''341    for c in source:342        if c in '\f\t':343            result += c344        else:345            result += ' '346    return result347 348 349def get_source_segment(source, node, *, padded=False):350    """Get source code segment of the *source* that generated *node*.351 352    If some location information (`lineno`, `end_lineno`, `col_offset`,353    or `end_col_offset`) is missing, return None.354 355    If *padded* is `True`, the first line of a multi-line statement will356    be padded with spaces to match its original position.357    """358    try:359        if node.end_lineno is None or node.end_col_offset is None:360            return None361        lineno = node.lineno - 1362        end_lineno = node.end_lineno - 1363        col_offset = node.col_offset364        end_col_offset = node.end_col_offset365    except AttributeError:366        return None367 368    lines = _splitlines_no_ff(source, maxlines=end_lineno+1)369    if end_lineno == lineno:370        return lines[lineno].encode()[col_offset:end_col_offset].decode()371 372    if padded:373        padding = _pad_whitespace(lines[lineno].encode()[:col_offset].decode())374    else:375        padding = ''376 377    first = padding + lines[lineno].encode()[col_offset:].decode()378    last = lines[end_lineno].encode()[:end_col_offset].decode()379    lines = lines[lineno+1:end_lineno]380 381    lines.insert(0, first)382    lines.append(last)383    return ''.join(lines)384 385 386def walk(node):387    """388    Recursively yield all descendant nodes in the tree starting at *node*389    (including *node* itself), in no specified order.  This is useful if you390    only want to modify nodes in place and don't care about the context.391    """392    from collections import deque393    todo = deque([node])394    while todo:395        node = todo.popleft()396        todo.extend(iter_child_nodes(node))397        yield node398 399 400def compare(401    a,402    b,403    /,404    *,405    compare_attributes=False,406):407    """Recursively compares two ASTs.408 409    compare_attributes affects whether AST attributes are considered410    in the comparison. If compare_attributes is False (default), then411    attributes are ignored. Otherwise they must all be equal. This412    option is useful to check whether the ASTs are structurally equal but413    might differ in whitespace or similar details.414    """415 416    sentinel = object()  # handle the possibility of a missing attribute/field417 418    def _compare(a, b):419        # Compare two fields on an AST object, which may themselves be420        # AST objects, lists of AST objects, or primitive ASDL types421        # like identifiers and constants.422        if isinstance(a, AST):423            return compare(424                a,425                b,426                compare_attributes=compare_attributes,427            )428        elif isinstance(a, list):429            # If a field is repeated, then both objects will represent430            # the value as a list.431            if len(a) != len(b):432                return False433            for a_item, b_item in zip(a, b):434                if not _compare(a_item, b_item):435                    return False436            else:437                return True438        else:439            return type(a) is type(b) and a == b440 441    def _compare_fields(a, b):442        if a._fields != b._fields:443            return False444        for field in a._fields:445            a_field = getattr(a, field, sentinel)446            b_field = getattr(b, field, sentinel)447            if a_field is sentinel and b_field is sentinel:448                # both nodes are missing a field at runtime449                continue450            if a_field is sentinel or b_field is sentinel:451                # one of the node is missing a field452                return False453            if not _compare(a_field, b_field):454                return False455        else:456            return True457 458    def _compare_attributes(a, b):459        if a._attributes != b._attributes:460            return False461        # Attributes are always ints.462        for attr in a._attributes:463            a_attr = getattr(a, attr, sentinel)464            b_attr = getattr(b, attr, sentinel)465            if a_attr is sentinel and b_attr is sentinel:466                # both nodes are missing an attribute at runtime467                continue468            if a_attr != b_attr:469                return False470        else:471            return True472 473    if type(a) is not type(b):474        return False475    if not _compare_fields(a, b):476        return False477    if compare_attributes and not _compare_attributes(a, b):478        return False479    return True480 481 482class NodeVisitor(object):483    """484    A node visitor base class that walks the abstract syntax tree and calls a485    visitor function for every node found.  This function may return a value486    which is forwarded by the `visit` method.487 488    This class is meant to be subclassed, with the subclass adding visitor489    methods.490 491    Per default the visitor functions for the nodes are ``'visit_'`` +492    class name of the node.  So a `TryFinally` node visit function would493    be `visit_TryFinally`.  This behavior can be changed by overriding494    the `visit` method.  If no visitor function exists for a node495    (return value `None`) the `generic_visit` visitor is used instead.496 497    Don't use the `NodeVisitor` if you want to apply changes to nodes during498    traversing.  For this a special visitor exists (`NodeTransformer`) that499    allows modifications.500    """501 502    def visit(self, node):503        """Visit a node."""504        method = 'visit_' + node.__class__.__name__505        visitor = getattr(self, method, self.generic_visit)506        return visitor(node)507 508    def generic_visit(self, node):509        """Called if no explicit visitor function exists for a node."""510        for field, value in iter_fields(node):511            if isinstance(value, list):512                for item in value:513                    if isinstance(item, AST):514                        self.visit(item)515            elif isinstance(value, AST):516                self.visit(value)517 518 519class NodeTransformer(NodeVisitor):520    """521    A :class:`NodeVisitor` subclass that walks the abstract syntax tree and522    allows modification of nodes.523 524    The `NodeTransformer` will walk the AST and use the return value of the525    visitor methods to replace or remove the old node.  If the return value of526    the visitor method is ``None``, the node will be removed from its location,527    otherwise it is replaced with the return value.  The return value may be the528    original node in which case no replacement takes place.529 530    Here is an example transformer that rewrites all occurrences of name lookups531    (``foo``) to ``data['foo']``::532 533       class RewriteName(NodeTransformer):534 535           def visit_Name(self, node):536               return Subscript(537                   value=Name(id='data', ctx=Load()),538                   slice=Constant(value=node.id),539                   ctx=node.ctx540               )541 542    Keep in mind that if the node you're operating on has child nodes you must543    either transform the child nodes yourself or call the :meth:`generic_visit`544    method for the node first.545 546    For nodes that were part of a collection of statements (that applies to all547    statement nodes), the visitor may also return a list of nodes rather than548    just a single node.549 550    Usually you use the transformer like this::551 552       node = YourTransformer().visit(node)553    """554 555    def generic_visit(self, node):556        for field, old_value in iter_fields(node):557            if isinstance(old_value, list):558                new_values = []559                for value in old_value:560                    if isinstance(value, AST):561                        value = self.visit(value)562                        if value is None:563                            continue564                        elif not isinstance(value, AST):565                            new_values.extend(value)566                            continue567                    new_values.append(value)568                old_value[:] = new_values569            elif isinstance(old_value, AST):570                new_node = self.visit(old_value)571                if new_node is None:572                    delattr(node, field)573                else:574                    setattr(node, field, new_node)575        return node576 577class slice(AST):578    """Deprecated AST node class."""579 580class Index(slice):581    """Deprecated AST node class. Use the index value directly instead."""582    def __new__(cls, value, **kwargs):583        return value584 585class ExtSlice(slice):586    """Deprecated AST node class. Use ast.Tuple instead."""587    def __new__(cls, dims=(), **kwargs):588        return Tuple(list(dims), Load(), **kwargs)589 590# If the ast module is loaded more than once, only add deprecated methods once591if not hasattr(Tuple, 'dims'):592    # The following code is for backward compatibility.593    # It will be removed in future.594 595    def _dims_getter(self):596        """Deprecated. Use elts instead."""597        return self.elts598 599    def _dims_setter(self, value):600        self.elts = value601 602    Tuple.dims = property(_dims_getter, _dims_setter)603 604class Suite(mod):605    """Deprecated AST node class.  Unused in Python 3."""606 607class AugLoad(expr_context):608    """Deprecated AST node class.  Unused in Python 3."""609 610class AugStore(expr_context):611    """Deprecated AST node class.  Unused in Python 3."""612 613class Param(expr_context):614    """Deprecated AST node class.  Unused in Python 3."""615 616 617def unparse(ast_obj):618    global _Unparser619    try:620        unparser = _Unparser()621    except NameError:622        from _ast_unparse import Unparser as _Unparser623        unparser = _Unparser()624    return unparser.visit(ast_obj)625 626 627def main(args=None):628    import argparse629    import sys630 631    parser = argparse.ArgumentParser(color=True)632    parser.add_argument('infile', nargs='?', default='-',633                        help='the file to parse; defaults to stdin')634    parser.add_argument('-m', '--mode', default='exec',635                        choices=('exec', 'single', 'eval', 'func_type'),636                        help='specify what kind of code must be parsed')637    parser.add_argument('--no-type-comments', default=True, action='store_false',638                        help="don't add information about type comments")639    parser.add_argument('-a', '--include-attributes', action='store_true',640                        help='include attributes such as line numbers and '641                             'column offsets')642    parser.add_argument('-i', '--indent', type=int, default=3,643                        help='indentation of nodes (number of spaces)')644    parser.add_argument('--feature-version',645                        type=str, default=None, metavar='VERSION',646                        help='Python version in the format 3.x '647                             '(for example, 3.10)')648    parser.add_argument('-O', '--optimize',649                        type=int, default=-1, metavar='LEVEL',650                        help='optimization level for parser (default -1)')651    parser.add_argument('--show-empty', default=False, action='store_true',652                        help='show empty lists and fields in dump output')653    args = parser.parse_args(args)654 655    if args.infile == '-':656        name = '<stdin>'657        source = sys.stdin.buffer.read()658    else:659        name = args.infile660        with open(args.infile, 'rb') as infile:661            source = infile.read()662 663    # Process feature_version664    feature_version = None665    if args.feature_version:666        try:667            major, minor = map(int, args.feature_version.split('.', 1))668        except ValueError:669            parser.error('Invalid format for --feature-version; '670                         'expected format 3.x (for example, 3.10)')671 672        feature_version = (major, minor)673 674    tree = parse(source, name, args.mode, type_comments=args.no_type_comments,675                 feature_version=feature_version, optimize=args.optimize)676    print(dump(tree, include_attributes=args.include_attributes,677               indent=args.indent, show_empty=args.show_empty))678 679if __name__ == '__main__':680    main()681