codekingpro/portable-devtools

1"""Parse tokens from the lexer into nodes for the compiler."""2 3import typing4import typing as t5 6from . import nodes7from .exceptions import TemplateAssertionError8from .exceptions import TemplateSyntaxError9from .lexer import describe_token10from .lexer import describe_token_expr11 12if t.TYPE_CHECKING:13    import typing_extensions as te14 15    from .environment import Environment16 17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)19 20_statement_keywords = frozenset(21    [22        "for",23        "if",24        "block",25        "extends",26        "print",27        "macro",28        "include",29        "from",30        "import",31        "set",32        "with",33        "autoescape",34    ]35)36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])37 38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {39    "add": nodes.Add,40    "sub": nodes.Sub,41    "mul": nodes.Mul,42    "div": nodes.Div,43    "floordiv": nodes.FloorDiv,44    "mod": nodes.Mod,45}46 47 48class Parser:49    """This is the central parsing class Jinja uses.  It's passed to50    extensions and can be used to parse expressions or statements.51    """52 53    def __init__(54        self,55        environment: "Environment",56        source: str,57        name: t.Optional[str] = None,58        filename: t.Optional[str] = None,59        state: t.Optional[str] = None,60    ) -> None:61        self.environment = environment62        self.stream = environment._tokenize(source, name, filename, state)63        self.name = name64        self.filename = filename65        self.closed = False66        self.extensions: t.Dict[67            str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]68        ] = {}69        for extension in environment.iter_extensions():70            for tag in extension.tags:71                self.extensions[tag] = extension.parse72        self._last_identifier = 073        self._tag_stack: t.List[str] = []74        self._end_token_stack: t.List[t.Tuple[str, ...]] = []75 76    def fail(77        self,78        msg: str,79        lineno: t.Optional[int] = None,80        exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,81    ) -> "te.NoReturn":82        """Convenience method that raises `exc` with the message, passed83        line number or last line number as well as the current name and84        filename.85        """86        if lineno is None:87            lineno = self.stream.current.lineno88        raise exc(msg, lineno, self.name, self.filename)89 90    def _fail_ut_eof(91        self,92        name: t.Optional[str],93        end_token_stack: t.List[t.Tuple[str, ...]],94        lineno: t.Optional[int],95    ) -> "te.NoReturn":96        expected: t.Set[str] = set()97        for exprs in end_token_stack:98            expected.update(map(describe_token_expr, exprs))99        if end_token_stack:100            currently_looking: t.Optional[str] = " or ".join(101                map(repr, map(describe_token_expr, end_token_stack[-1]))102            )103        else:104            currently_looking = None105 106        if name is None:107            message = ["Unexpected end of template."]108        else:109            message = [f"Encountered unknown tag {name!r}."]110 111        if currently_looking:112            if name is not None and name in expected:113                message.append(114                    "You probably made a nesting mistake. Jinja is expecting this tag,"115                    f" but currently looking for {currently_looking}."116                )117            else:118                message.append(119                    f"Jinja was looking for the following tags: {currently_looking}."120                )121 122        if self._tag_stack:123            message.append(124                "The innermost block that needs to be closed is"125                f" {self._tag_stack[-1]!r}."126            )127 128        self.fail(" ".join(message), lineno)129 130    def fail_unknown_tag(131        self, name: str, lineno: t.Optional[int] = None132    ) -> "te.NoReturn":133        """Called if the parser encounters an unknown tag.  Tries to fail134        with a human readable error message that could help to identify135        the problem.136        """137        self._fail_ut_eof(name, self._end_token_stack, lineno)138 139    def fail_eof(140        self,141        end_tokens: t.Optional[t.Tuple[str, ...]] = None,142        lineno: t.Optional[int] = None,143    ) -> "te.NoReturn":144        """Like fail_unknown_tag but for end of template situations."""145        stack = list(self._end_token_stack)146        if end_tokens is not None:147            stack.append(end_tokens)148        self._fail_ut_eof(None, stack, lineno)149 150    def is_tuple_end(151        self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None152    ) -> bool:153        """Are we at the end of a tuple?"""154        if self.stream.current.type in ("variable_end", "block_end", "rparen"):155            return True156        elif extra_end_rules is not None:157            return self.stream.current.test_any(extra_end_rules)  # type: ignore158        return False159 160    def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:161        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""162        self._last_identifier += 1163        rv = object.__new__(nodes.InternalName)164        nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)165        return rv166 167    def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:168        """Parse a single statement."""169        token = self.stream.current170        if token.type != "name":171            self.fail("tag name expected", token.lineno)172        self._tag_stack.append(token.value)173        pop_tag = True174        try:175            if token.value in _statement_keywords:176                f = getattr(self, f"parse_{self.stream.current.value}")177                return f()  # type: ignore178            if token.value == "call":179                return self.parse_call_block()180            if token.value == "filter":181                return self.parse_filter_block()182            ext = self.extensions.get(token.value)183            if ext is not None:184                return ext(self)185 186            # did not work out, remove the token we pushed by accident187            # from the stack so that the unknown tag fail function can188            # produce a proper error message.189            self._tag_stack.pop()190            pop_tag = False191            self.fail_unknown_tag(token.value, token.lineno)192        finally:193            if pop_tag:194                self._tag_stack.pop()195 196    def parse_statements(197        self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False198    ) -> t.List[nodes.Node]:199        """Parse multiple statements into a list until one of the end tokens200        is reached.  This is used to parse the body of statements as it also201        parses template data if appropriate.  The parser checks first if the202        current token is a colon and skips it if there is one.  Then it checks203        for the block end and parses until if one of the `end_tokens` is204        reached.  Per default the active token in the stream at the end of205        the call is the matched end token.  If this is not wanted `drop_needle`206        can be set to `True` and the end token is removed.207        """208        # the first token may be a colon for python compatibility209        self.stream.skip_if("colon")210 211        # in the future it would be possible to add whole code sections212        # by adding some sort of end of statement token and parsing those here.213        self.stream.expect("block_end")214        result = self.subparse(end_tokens)215 216        # we reached the end of the template too early, the subparser217        # does not check for this, so we do that now218        if self.stream.current.type == "eof":219            self.fail_eof(end_tokens)220 221        if drop_needle:222            next(self.stream)223        return result224 225    def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:226        """Parse an assign statement."""227        lineno = next(self.stream).lineno228        target = self.parse_assign_target(with_namespace=True)229        if self.stream.skip_if("assign"):230            expr = self.parse_tuple()231            return nodes.Assign(target, expr, lineno=lineno)232        filter_node = self.parse_filter(None)233        body = self.parse_statements(("name:endset",), drop_needle=True)234        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)235 236    def parse_for(self) -> nodes.For:237        """Parse a for loop."""238        lineno = self.stream.expect("name:for").lineno239        target = self.parse_assign_target(extra_end_rules=("name:in",))240        self.stream.expect("name:in")241        iter = self.parse_tuple(242            with_condexpr=False, extra_end_rules=("name:recursive",)243        )244        test = None245        if self.stream.skip_if("name:if"):246            test = self.parse_expression()247        recursive = self.stream.skip_if("name:recursive")248        body = self.parse_statements(("name:endfor", "name:else"))249        if next(self.stream).value == "endfor":250            else_ = []251        else:252            else_ = self.parse_statements(("name:endfor",), drop_needle=True)253        return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)254 255    def parse_if(self) -> nodes.If:256        """Parse an if construct."""257        node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)258        while True:259            node.test = self.parse_tuple(with_condexpr=False)260            node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))261            node.elif_ = []262            node.else_ = []263            token = next(self.stream)264            if token.test("name:elif"):265                node = nodes.If(lineno=self.stream.current.lineno)266                result.elif_.append(node)267                continue268            elif token.test("name:else"):269                result.else_ = self.parse_statements(("name:endif",), drop_needle=True)270            break271        return result272 273    def parse_with(self) -> nodes.With:274        node = nodes.With(lineno=next(self.stream).lineno)275        targets: t.List[nodes.Expr] = []276        values: t.List[nodes.Expr] = []277        while self.stream.current.type != "block_end":278            if targets:279                self.stream.expect("comma")280            target = self.parse_assign_target()281            target.set_ctx("param")282            targets.append(target)283            self.stream.expect("assign")284            values.append(self.parse_expression())285        node.targets = targets286        node.values = values287        node.body = self.parse_statements(("name:endwith",), drop_needle=True)288        return node289 290    def parse_autoescape(self) -> nodes.Scope:291        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)292        node.options = [nodes.Keyword("autoescape", self.parse_expression())]293        node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)294        return nodes.Scope([node])295 296    def parse_block(self) -> nodes.Block:297        node = nodes.Block(lineno=next(self.stream).lineno)298        node.name = self.stream.expect("name").value299        node.scoped = self.stream.skip_if("name:scoped")300        node.required = self.stream.skip_if("name:required")301 302        # common problem people encounter when switching from django303        # to jinja.  we do not support hyphens in block names, so let's304        # raise a nicer error message in that case.305        if self.stream.current.type == "sub":306            self.fail(307                "Block names in Jinja have to be valid Python identifiers and may not"308                " contain hyphens, use an underscore instead."309            )310 311        node.body = self.parse_statements(("name:endblock",), drop_needle=True)312 313        # enforce that required blocks only contain whitespace or comments314        # by asserting that the body, if not empty, is just TemplateData nodes315        # with whitespace data316        if node.required:317            for body_node in node.body:318                if not isinstance(body_node, nodes.Output) or any(319                    not isinstance(output_node, nodes.TemplateData)320                    or not output_node.data.isspace()321                    for output_node in body_node.nodes322                ):323                    self.fail("Required blocks can only contain comments or whitespace")324 325        self.stream.skip_if("name:" + node.name)326        return node327 328    def parse_extends(self) -> nodes.Extends:329        node = nodes.Extends(lineno=next(self.stream).lineno)330        node.template = self.parse_expression()331        return node332 333    def parse_import_context(334        self, node: _ImportInclude, default: bool335    ) -> _ImportInclude:336        if self.stream.current.test_any(337            "name:with", "name:without"338        ) and self.stream.look().test("name:context"):339            node.with_context = next(self.stream).value == "with"340            self.stream.skip()341        else:342            node.with_context = default343        return node344 345    def parse_include(self) -> nodes.Include:346        node = nodes.Include(lineno=next(self.stream).lineno)347        node.template = self.parse_expression()348        if self.stream.current.test("name:ignore") and self.stream.look().test(349            "name:missing"350        ):351            node.ignore_missing = True352            self.stream.skip(2)353        else:354            node.ignore_missing = False355        return self.parse_import_context(node, True)356 357    def parse_import(self) -> nodes.Import:358        node = nodes.Import(lineno=next(self.stream).lineno)359        node.template = self.parse_expression()360        self.stream.expect("name:as")361        node.target = self.parse_assign_target(name_only=True).name362        return self.parse_import_context(node, False)363 364    def parse_from(self) -> nodes.FromImport:365        node = nodes.FromImport(lineno=next(self.stream).lineno)366        node.template = self.parse_expression()367        self.stream.expect("name:import")368        node.names = []369 370        def parse_context() -> bool:371            if self.stream.current.value in {372                "with",373                "without",374            } and self.stream.look().test("name:context"):375                node.with_context = next(self.stream).value == "with"376                self.stream.skip()377                return True378            return False379 380        while True:381            if node.names:382                self.stream.expect("comma")383            if self.stream.current.type == "name":384                if parse_context():385                    break386                target = self.parse_assign_target(name_only=True)387                if target.name.startswith("_"):388                    self.fail(389                        "names starting with an underline can not be imported",390                        target.lineno,391                        exc=TemplateAssertionError,392                    )393                if self.stream.skip_if("name:as"):394                    alias = self.parse_assign_target(name_only=True)395                    node.names.append((target.name, alias.name))396                else:397                    node.names.append(target.name)398                if parse_context() or self.stream.current.type != "comma":399                    break400            else:401                self.stream.expect("name")402        if not hasattr(node, "with_context"):403            node.with_context = False404        return node405 406    def parse_signature(self, node: _MacroCall) -> None:407        args = node.args = []408        defaults = node.defaults = []409        self.stream.expect("lparen")410        while self.stream.current.type != "rparen":411            if args:412                self.stream.expect("comma")413            arg = self.parse_assign_target(name_only=True)414            arg.set_ctx("param")415            if self.stream.skip_if("assign"):416                defaults.append(self.parse_expression())417            elif defaults:418                self.fail("non-default argument follows default argument")419            args.append(arg)420        self.stream.expect("rparen")421 422    def parse_call_block(self) -> nodes.CallBlock:423        node = nodes.CallBlock(lineno=next(self.stream).lineno)424        if self.stream.current.type == "lparen":425            self.parse_signature(node)426        else:427            node.args = []428            node.defaults = []429 430        call_node = self.parse_expression()431        if not isinstance(call_node, nodes.Call):432            self.fail("expected call", node.lineno)433        node.call = call_node434        node.body = self.parse_statements(("name:endcall",), drop_needle=True)435        return node436 437    def parse_filter_block(self) -> nodes.FilterBlock:438        node = nodes.FilterBlock(lineno=next(self.stream).lineno)439        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore440        node.body = self.parse_statements(("name:endfilter",), drop_needle=True)441        return node442 443    def parse_macro(self) -> nodes.Macro:444        node = nodes.Macro(lineno=next(self.stream).lineno)445        node.name = self.parse_assign_target(name_only=True).name446        self.parse_signature(node)447        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)448        return node449 450    def parse_print(self) -> nodes.Output:451        node = nodes.Output(lineno=next(self.stream).lineno)452        node.nodes = []453        while self.stream.current.type != "block_end":454            if node.nodes:455                self.stream.expect("comma")456            node.nodes.append(self.parse_expression())457        return node458 459    @typing.overload460    def parse_assign_target(461        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...462    ) -> nodes.Name: ...463 464    @typing.overload465    def parse_assign_target(466        self,467        with_tuple: bool = True,468        name_only: bool = False,469        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,470        with_namespace: bool = False,471    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...472 473    def parse_assign_target(474        self,475        with_tuple: bool = True,476        name_only: bool = False,477        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,478        with_namespace: bool = False,479    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:480        """Parse an assignment target.  As Jinja allows assignments to481        tuples, this function can parse all allowed assignment targets.  Per482        default assignments to tuples are parsed, that can be disable however483        by setting `with_tuple` to `False`.  If only assignments to names are484        wanted `name_only` can be set to `True`.  The `extra_end_rules`485        parameter is forwarded to the tuple parsing function.  If486        `with_namespace` is enabled, a namespace assignment may be parsed.487        """488        target: nodes.Expr489 490        if with_namespace and self.stream.look().type == "dot":491            token = self.stream.expect("name")492            next(self.stream)  # dot493            attr = self.stream.expect("name")494            target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)495        elif name_only:496            token = self.stream.expect("name")497            target = nodes.Name(token.value, "store", lineno=token.lineno)498        else:499            if with_tuple:500                target = self.parse_tuple(501                    simplified=True, extra_end_rules=extra_end_rules502                )503            else:504                target = self.parse_primary()505 506            target.set_ctx("store")507 508        if not target.can_assign():509            self.fail(510                f"can't assign to {type(target).__name__.lower()!r}", target.lineno511            )512 513        return target  # type: ignore514 515    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:516        """Parse an expression.  Per default all expressions are parsed, if517        the optional `with_condexpr` parameter is set to `False` conditional518        expressions are not parsed.519        """520        if with_condexpr:521            return self.parse_condexpr()522        return self.parse_or()523 524    def parse_condexpr(self) -> nodes.Expr:525        lineno = self.stream.current.lineno526        expr1 = self.parse_or()527        expr3: t.Optional[nodes.Expr]528 529        while self.stream.skip_if("name:if"):530            expr2 = self.parse_or()531            if self.stream.skip_if("name:else"):532                expr3 = self.parse_condexpr()533            else:534                expr3 = None535            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)536            lineno = self.stream.current.lineno537        return expr1538 539    def parse_or(self) -> nodes.Expr:540        lineno = self.stream.current.lineno541        left = self.parse_and()542        while self.stream.skip_if("name:or"):543            right = self.parse_and()544            left = nodes.Or(left, right, lineno=lineno)545            lineno = self.stream.current.lineno546        return left547 548    def parse_and(self) -> nodes.Expr:549        lineno = self.stream.current.lineno550        left = self.parse_not()551        while self.stream.skip_if("name:and"):552            right = self.parse_not()553            left = nodes.And(left, right, lineno=lineno)554            lineno = self.stream.current.lineno555        return left556 557    def parse_not(self) -> nodes.Expr:558        if self.stream.current.test("name:not"):559            lineno = next(self.stream).lineno560            return nodes.Not(self.parse_not(), lineno=lineno)561        return self.parse_compare()562 563    def parse_compare(self) -> nodes.Expr:564        lineno = self.stream.current.lineno565        expr = self.parse_math1()566        ops = []567        while True:568            token_type = self.stream.current.type569            if token_type in _compare_operators:570                next(self.stream)571                ops.append(nodes.Operand(token_type, self.parse_math1()))572            elif self.stream.skip_if("name:in"):573                ops.append(nodes.Operand("in", self.parse_math1()))574            elif self.stream.current.test("name:not") and self.stream.look().test(575                "name:in"576            ):577                self.stream.skip(2)578                ops.append(nodes.Operand("notin", self.parse_math1()))579            else:580                break581            lineno = self.stream.current.lineno582        if not ops:583            return expr584        return nodes.Compare(expr, ops, lineno=lineno)585 586    def parse_math1(self) -> nodes.Expr:587        lineno = self.stream.current.lineno588        left = self.parse_concat()589        while self.stream.current.type in ("add", "sub"):590            cls = _math_nodes[self.stream.current.type]591            next(self.stream)592            right = self.parse_concat()593            left = cls(left, right, lineno=lineno)594            lineno = self.stream.current.lineno595        return left596 597    def parse_concat(self) -> nodes.Expr:598        lineno = self.stream.current.lineno599        args = [self.parse_math2()]600        while self.stream.current.type == "tilde":601            next(self.stream)602            args.append(self.parse_math2())603        if len(args) == 1:604            return args[0]605        return nodes.Concat(args, lineno=lineno)606 607    def parse_math2(self) -> nodes.Expr:608        lineno = self.stream.current.lineno609        left = self.parse_pow()610        while self.stream.current.type in ("mul", "div", "floordiv", "mod"):611            cls = _math_nodes[self.stream.current.type]612            next(self.stream)613            right = self.parse_pow()614            left = cls(left, right, lineno=lineno)615            lineno = self.stream.current.lineno616        return left617 618    def parse_pow(self) -> nodes.Expr:619        lineno = self.stream.current.lineno620        left = self.parse_unary()621        while self.stream.current.type == "pow":622            next(self.stream)623            right = self.parse_unary()624            left = nodes.Pow(left, right, lineno=lineno)625            lineno = self.stream.current.lineno626        return left627 628    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:629        token_type = self.stream.current.type630        lineno = self.stream.current.lineno631        node: nodes.Expr632 633        if token_type == "sub":634            next(self.stream)635            node = nodes.Neg(self.parse_unary(False), lineno=lineno)636        elif token_type == "add":637            next(self.stream)638            node = nodes.Pos(self.parse_unary(False), lineno=lineno)639        else:640            node = self.parse_primary()641        node = self.parse_postfix(node)642        if with_filter:643            node = self.parse_filter_expr(node)644        return node645 646    def parse_primary(self) -> nodes.Expr:647        token = self.stream.current648        node: nodes.Expr649        if token.type == "name":650            if token.value in ("true", "false", "True", "False"):651                node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)652            elif token.value in ("none", "None"):653                node = nodes.Const(None, lineno=token.lineno)654            else:655                node = nodes.Name(token.value, "load", lineno=token.lineno)656            next(self.stream)657        elif token.type == "string":658            next(self.stream)659            buf = [token.value]660            lineno = token.lineno661            while self.stream.current.type == "string":662                buf.append(self.stream.current.value)663                next(self.stream)664            node = nodes.Const("".join(buf), lineno=lineno)665        elif token.type in ("integer", "float"):666            next(self.stream)667            node = nodes.Const(token.value, lineno=token.lineno)668        elif token.type == "lparen":669            next(self.stream)670            node = self.parse_tuple(explicit_parentheses=True)671            self.stream.expect("rparen")672        elif token.type == "lbracket":673            node = self.parse_list()674        elif token.type == "lbrace":675            node = self.parse_dict()676        else:677            self.fail(f"unexpected {describe_token(token)!r}", token.lineno)678        return node679 680    def parse_tuple(681        self,682        simplified: bool = False,683        with_condexpr: bool = True,684        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,685        explicit_parentheses: bool = False,686    ) -> t.Union[nodes.Tuple, nodes.Expr]:687        """Works like `parse_expression` but if multiple expressions are688        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.689        This method could also return a regular expression instead of a tuple690        if no commas where found.691 692        The default parsing mode is a full tuple.  If `simplified` is `True`693        only names and literals are parsed.  The `no_condexpr` parameter is694        forwarded to :meth:`parse_expression`.695 696        Because tuples do not require delimiters and may end in a bogus comma697        an extra hint is needed that marks the end of a tuple.  For example698        for loops support tuples between `for` and `in`.  In that case the699        `extra_end_rules` is set to ``['name:in']``.700 701        `explicit_parentheses` is true if the parsing was triggered by an702        expression in parentheses.  This is used to figure out if an empty703        tuple is a valid expression or not.704        """705        lineno = self.stream.current.lineno706        if simplified:707            parse = self.parse_primary708        elif with_condexpr:709            parse = self.parse_expression710        else:711 712            def parse() -> nodes.Expr:713                return self.parse_expression(with_condexpr=False)714 715        args: t.List[nodes.Expr] = []716        is_tuple = False717 718        while True:719            if args:720                self.stream.expect("comma")721            if self.is_tuple_end(extra_end_rules):722                break723            args.append(parse())724            if self.stream.current.type == "comma":725                is_tuple = True726            else:727                break728            lineno = self.stream.current.lineno729 730        if not is_tuple:731            if args:732                return args[0]733 734            # if we don't have explicit parentheses, an empty tuple is735            # not a valid expression.  This would mean nothing (literally736            # nothing) in the spot of an expression would be an empty737            # tuple.738            if not explicit_parentheses:739                self.fail(740                    "Expected an expression,"741                    f" got {describe_token(self.stream.current)!r}"742                )743 744        return nodes.Tuple(args, "load", lineno=lineno)745 746    def parse_list(self) -> nodes.List:747        token = self.stream.expect("lbracket")748        items: t.List[nodes.Expr] = []749        while self.stream.current.type != "rbracket":750            if items:751                self.stream.expect("comma")752            if self.stream.current.type == "rbracket":753                break754            items.append(self.parse_expression())755        self.stream.expect("rbracket")756        return nodes.List(items, lineno=token.lineno)757 758    def parse_dict(self) -> nodes.Dict:759        token = self.stream.expect("lbrace")760        items: t.List[nodes.Pair] = []761        while self.stream.current.type != "rbrace":762            if items:763                self.stream.expect("comma")764            if self.stream.current.type == "rbrace":765                break766            key = self.parse_expression()767            self.stream.expect("colon")768            value = self.parse_expression()769            items.append(nodes.Pair(key, value, lineno=key.lineno))770        self.stream.expect("rbrace")771        return nodes.Dict(items, lineno=token.lineno)772 773    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:774        while True:775            token_type = self.stream.current.type776            if token_type == "dot" or token_type == "lbracket":777                node = self.parse_subscript(node)778            # calls are valid both after postfix expressions (getattr779            # and getitem) as well as filters and tests780            elif token_type == "lparen":781                node = self.parse_call(node)782            else:783                break784        return node785 786    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:787        while True:788            token_type = self.stream.current.type789            if token_type == "pipe":790                node = self.parse_filter(node)  # type: ignore791            elif token_type == "name" and self.stream.current.value == "is":792                node = self.parse_test(node)793            # calls are valid both after postfix expressions (getattr794            # and getitem) as well as filters and tests795            elif token_type == "lparen":796                node = self.parse_call(node)797            else:798                break799        return node800 801    def parse_subscript(802        self, node: nodes.Expr803    ) -> t.Union[nodes.Getattr, nodes.Getitem]:804        token = next(self.stream)805        arg: nodes.Expr806 807        if token.type == "dot":808            attr_token = self.stream.current809            next(self.stream)810            if attr_token.type == "name":811                return nodes.Getattr(812                    node, attr_token.value, "load", lineno=token.lineno813                )814            elif attr_token.type != "integer":815                self.fail("expected name or number", attr_token.lineno)816            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)817            return nodes.Getitem(node, arg, "load", lineno=token.lineno)818        if token.type == "lbracket":819            args: t.List[nodes.Expr] = []820            while self.stream.current.type != "rbracket":821                if args:822                    self.stream.expect("comma")823                args.append(self.parse_subscribed())824            self.stream.expect("rbracket")825            if len(args) == 1:826                arg = args[0]827            else:828                arg = nodes.Tuple(args, "load", lineno=token.lineno)829            return nodes.Getitem(node, arg, "load", lineno=token.lineno)830        self.fail("expected subscript expression", token.lineno)831 832    def parse_subscribed(self) -> nodes.Expr:833        lineno = self.stream.current.lineno834        args: t.List[t.Optional[nodes.Expr]]835 836        if self.stream.current.type == "colon":837            next(self.stream)838            args = [None]839        else:840            node = self.parse_expression()841            if self.stream.current.type != "colon":842                return node843            next(self.stream)844            args = [node]845 846        if self.stream.current.type == "colon":847            args.append(None)848        elif self.stream.current.type not in ("rbracket", "comma"):849            args.append(self.parse_expression())850        else:851            args.append(None)852 853        if self.stream.current.type == "colon":854            next(self.stream)855            if self.stream.current.type not in ("rbracket", "comma"):856                args.append(self.parse_expression())857            else:858                args.append(None)859        else:860            args.append(None)861 862        return nodes.Slice(lineno=lineno, *args)  # noqa: B026863 864    def parse_call_args(865        self,866    ) -> t.Tuple[867        t.List[nodes.Expr],868        t.List[nodes.Keyword],869        t.Optional[nodes.Expr],870        t.Optional[nodes.Expr],871    ]:872        token = self.stream.expect("lparen")873        args = []874        kwargs = []875        dyn_args = None876        dyn_kwargs = None877        require_comma = False878 879        def ensure(expr: bool) -> None:880            if not expr:881                self.fail("invalid syntax for function call expression", token.lineno)882 883        while self.stream.current.type != "rparen":884            if require_comma:885                self.stream.expect("comma")886 887                # support for trailing comma888                if self.stream.current.type == "rparen":889                    break890 891            if self.stream.current.type == "mul":892                ensure(dyn_args is None and dyn_kwargs is None)893                next(self.stream)894                dyn_args = self.parse_expression()895            elif self.stream.current.type == "pow":896                ensure(dyn_kwargs is None)897                next(self.stream)898                dyn_kwargs = self.parse_expression()899            else:900                if (901                    self.stream.current.type == "name"902                    and self.stream.look().type == "assign"903                ):904                    # Parsing a kwarg905                    ensure(dyn_kwargs is None)906                    key = self.stream.current.value907                    self.stream.skip(2)908                    value = self.parse_expression()909                    kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))910                else:911                    # Parsing an arg912                    ensure(dyn_args is None and dyn_kwargs is None and not kwargs)913                    args.append(self.parse_expression())914 915            require_comma = True916 917        self.stream.expect("rparen")918        return args, kwargs, dyn_args, dyn_kwargs919 920    def parse_call(self, node: nodes.Expr) -> nodes.Call:921        # The lparen will be expected in parse_call_args, but the lineno922        # needs to be recorded before the stream is advanced.923        token = self.stream.current924        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()925        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)926 927    def parse_filter(928        self, node: t.Optional[nodes.Expr], start_inline: bool = False929    ) -> t.Optional[nodes.Expr]:930        while self.stream.current.type == "pipe" or start_inline:931            if not start_inline:932                next(self.stream)933            token = self.stream.expect("name")934            name = token.value935            while self.stream.current.type == "dot":936                next(self.stream)937                name += "." + self.stream.expect("name").value938            if self.stream.current.type == "lparen":939                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()940            else:941                args = []942                kwargs = []943                dyn_args = dyn_kwargs = None944            node = nodes.Filter(945                node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno946            )947            start_inline = False948        return node949 950    def parse_test(self, node: nodes.Expr) -> nodes.Expr:951        token = next(self.stream)952        if self.stream.current.test("name:not"):953            next(self.stream)954            negated = True955        else:956            negated = False957        name = self.stream.expect("name").value958        while self.stream.current.type == "dot":959            next(self.stream)960            name += "." + self.stream.expect("name").value961        dyn_args = dyn_kwargs = None962        kwargs: t.List[nodes.Keyword] = []963        if self.stream.current.type == "lparen":964            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()965        elif self.stream.current.type in {966            "name",967            "string",968            "integer",969            "float",970            "lparen",971            "lbracket",972            "lbrace",973        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):974            if self.stream.current.test("name:is"):975                self.fail("You cannot chain multiple tests with is")976            arg_node = self.parse_primary()977            arg_node = self.parse_postfix(arg_node)978            args = [arg_node]979        else:980            args = []981        node = nodes.Test(982            node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno983        )984        if negated:985            node = nodes.Not(node, lineno=token.lineno)986        return node987 988    def subparse(989        self, end_tokens: t.Optional[t.Tuple[str, ...]] = None990    ) -> t.List[nodes.Node]:991        body: t.List[nodes.Node] = []992        data_buffer: t.List[nodes.Node] = []993        add_data = data_buffer.append994 995        if end_tokens is not None:996            self._end_token_stack.append(end_tokens)997 998        def flush_data() -> None:999            if data_buffer:1000                lineno = data_buffer[0].lineno1001                body.append(nodes.Output(data_buffer[:], lineno=lineno))1002                del data_buffer[:]1003 1004        try:1005            while self.stream:1006                token = self.stream.current1007                if token.type == "data":1008                    if token.value:1009                        add_data(nodes.TemplateData(token.value, lineno=token.lineno))1010                    next(self.stream)1011                elif token.type == "variable_begin":1012                    next(self.stream)1013                    add_data(self.parse_tuple(with_condexpr=True))1014                    self.stream.expect("variable_end")1015                elif token.type == "block_begin":1016                    flush_data()1017                    next(self.stream)1018                    if end_tokens is not None and self.stream.current.test_any(1019                        *end_tokens1020                    ):1021                        return body1022                    rv = self.parse_statement()1023                    if isinstance(rv, list):1024                        body.extend(rv)1025                    else:1026                        body.append(rv)1027                    self.stream.expect("block_end")1028                else:1029                    raise AssertionError("internal parsing error")1030 1031            flush_data()1032        finally:1033            if end_tokens is not None:1034                self._end_token_stack.pop()1035        return body1036 1037    def parse(self) -> nodes.Template:1038        """Parse the whole template into a `Template` node."""1039        result = nodes.Template(self.subparse(), lineno=1)1040        result.set_environment(self.environment)1041        return result1042