codekingpro/portable-devtools

1"""Parse tokens from the lexer into nodes for the compiler."""2 3import typing4import typing as t5 6from . import nodes7from .exceptions import TemplateAssertionError8from .exceptions import TemplateSyntaxError9from .lexer import describe_token10from .lexer import describe_token_expr11 12if t.TYPE_CHECKING:13    import typing_extensions as te14 15    from .environment import Environment16 17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)19 20_statement_keywords = frozenset(21    [22        "for",23        "if",24        "block",25        "extends",26        "print",27        "macro",28        "include",29        "from",30        "import",31        "set",32        "with",33        "autoescape",34    ]35)36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])37 38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {39    "add": nodes.Add,40    "sub": nodes.Sub,41    "mul": nodes.Mul,42    "div": nodes.Div,43    "floordiv": nodes.FloorDiv,44    "mod": nodes.Mod,45}46 47 48class Parser:49    """This is the central parsing class Jinja uses.  It's passed to50    extensions and can be used to parse expressions or statements.51    """52 53    def __init__(54        self,55        environment: "Environment",56        source: str,57        name: t.Optional[str] = None,58        filename: t.Optional[str] = None,59        state: t.Optional[str] = None,60    ) -> None:61        self.environment = environment62        self.stream = environment._tokenize(source, name, filename, state)63        self.name = name64        self.filename = filename65        self.closed = False66        self.extensions: t.Dict[67            str, t.Callable[[Parser], t.Union[nodes.Node, t.List[nodes.Node]]]68        ] = {}69        for extension in environment.iter_extensions():70            for tag in extension.tags:71                self.extensions[tag] = extension.parse72        self._last_identifier = 073        self._tag_stack: t.List[str] = []74        self._end_token_stack: t.List[t.Tuple[str, ...]] = []75 76    def fail(77        self,78        msg: str,79        lineno: t.Optional[int] = None,80        exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,81    ) -> "te.NoReturn":82        """Convenience method that raises `exc` with the message, passed83        line number or last line number as well as the current name and84        filename.85        """86        if lineno is None:87            lineno = self.stream.current.lineno88        raise exc(msg, lineno, self.name, self.filename)89 90    def _fail_ut_eof(91        self,92        name: t.Optional[str],93        end_token_stack: t.List[t.Tuple[str, ...]],94        lineno: t.Optional[int],95    ) -> "te.NoReturn":96        expected: t.Set[str] = set()97        for exprs in end_token_stack:98            expected.update(map(describe_token_expr, exprs))99        if end_token_stack:100            currently_looking: t.Optional[str] = " or ".join(101                map(repr, map(describe_token_expr, end_token_stack[-1]))102            )103        else:104            currently_looking = None105 106        if name is None:107            message = ["Unexpected end of template."]108        else:109            message = [f"Encountered unknown tag {name!r}."]110 111        if currently_looking:112            if name is not None and name in expected:113                message.append(114                    "You probably made a nesting mistake. Jinja is expecting this tag,"115                    f" but currently looking for {currently_looking}."116                )117            else:118                message.append(119                    f"Jinja was looking for the following tags: {currently_looking}."120                )121 122        if self._tag_stack:123            message.append(124                "The innermost block that needs to be closed is"125                f" {self._tag_stack[-1]!r}."126            )127 128        self.fail(" ".join(message), lineno)129 130    def fail_unknown_tag(131        self, name: str, lineno: t.Optional[int] = None132    ) -> "te.NoReturn":133        """Called if the parser encounters an unknown tag.  Tries to fail134        with a human readable error message that could help to identify135        the problem.136        """137        self._fail_ut_eof(name, self._end_token_stack, lineno)138 139    def fail_eof(140        self,141        end_tokens: t.Optional[t.Tuple[str, ...]] = None,142        lineno: t.Optional[int] = None,143    ) -> "te.NoReturn":144        """Like fail_unknown_tag but for end of template situations."""145        stack = list(self._end_token_stack)146        if end_tokens is not None:147            stack.append(end_tokens)148        self._fail_ut_eof(None, stack, lineno)149 150    def is_tuple_end(151        self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None152    ) -> bool:153        """Are we at the end of a tuple?"""154        if self.stream.current.type in ("variable_end", "block_end", "rparen"):155            return True156        elif extra_end_rules is not None:157            return self.stream.current.test_any(extra_end_rules)  # type: ignore158        return False159 160    def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:161        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""162        self._last_identifier += 1163        rv = object.__new__(nodes.InternalName)164        nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)165        return rv166 167    def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:168        """Parse a single statement."""169        token = self.stream.current170        if token.type != "name":171            self.fail("tag name expected", token.lineno)172        self._tag_stack.append(token.value)173        pop_tag = True174        try:175            if token.value in _statement_keywords:176                f = getattr(self, f"parse_{self.stream.current.value}")177                return f()  # type: ignore178            if token.value == "call":179                return self.parse_call_block()180            if token.value == "filter":181                return self.parse_filter_block()182            ext = self.extensions.get(token.value)183            if ext is not None:184                return ext(self)185 186            # did not work out, remove the token we pushed by accident187            # from the stack so that the unknown tag fail function can188            # produce a proper error message.189            self._tag_stack.pop()190            pop_tag = False191            self.fail_unknown_tag(token.value, token.lineno)192        finally:193            if pop_tag:194                self._tag_stack.pop()195 196    def parse_statements(197        self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False198    ) -> t.List[nodes.Node]:199        """Parse multiple statements into a list until one of the end tokens200        is reached.  This is used to parse the body of statements as it also201        parses template data if appropriate.  The parser checks first if the202        current token is a colon and skips it if there is one.  Then it checks203        for the block end and parses until if one of the `end_tokens` is204        reached.  Per default the active token in the stream at the end of205        the call is the matched end token.  If this is not wanted `drop_needle`206        can be set to `True` and the end token is removed.207        """208        # the first token may be a colon for python compatibility209        self.stream.skip_if("colon")210 211        # in the future it would be possible to add whole code sections212        # by adding some sort of end of statement token and parsing those here.213        self.stream.expect("block_end")214        result = self.subparse(end_tokens)215 216        # we reached the end of the template too early, the subparser217        # does not check for this, so we do that now218        if self.stream.current.type == "eof":219            self.fail_eof(end_tokens)220 221        if drop_needle:222            next(self.stream)223        return result224 225    def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:226        """Parse an assign statement."""227        lineno = next(self.stream).lineno228        target = self.parse_assign_target(with_namespace=True)229        if self.stream.skip_if("assign"):230            expr = self.parse_tuple()231            return nodes.Assign(target, expr, lineno=lineno)232        filter_node = self.parse_filter(None)233        body = self.parse_statements(("name:endset",), drop_needle=True)234        return nodes.AssignBlock(target, filter_node, body, lineno=lineno)235 236    def parse_for(self) -> nodes.For:237        """Parse a for loop."""238        lineno = self.stream.expect("name:for").lineno239        target = self.parse_assign_target(extra_end_rules=("name:in",))240        self.stream.expect("name:in")241        iter = self.parse_tuple(242            with_condexpr=False, extra_end_rules=("name:recursive",)243        )244        test = None245        if self.stream.skip_if("name:if"):246            test = self.parse_expression()247        recursive = self.stream.skip_if("name:recursive")248        body = self.parse_statements(("name:endfor", "name:else"))249        if next(self.stream).value == "endfor":250            else_ = []251        else:252            else_ = self.parse_statements(("name:endfor",), drop_needle=True)253        return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)254 255    def parse_if(self) -> nodes.If:256        """Parse an if construct."""257        node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)258        while True:259            node.test = self.parse_tuple(with_condexpr=False)260            node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))261            node.elif_ = []262            node.else_ = []263            token = next(self.stream)264            if token.test("name:elif"):265                node = nodes.If(lineno=self.stream.current.lineno)266                result.elif_.append(node)267                continue268            elif token.test("name:else"):269                result.else_ = self.parse_statements(("name:endif",), drop_needle=True)270            break271        return result272 273    def parse_with(self) -> nodes.With:274        node = nodes.With(lineno=next(self.stream).lineno)275        targets: t.List[nodes.Expr] = []276        values: t.List[nodes.Expr] = []277        while self.stream.current.type != "block_end":278            if targets:279                self.stream.expect("comma")280            target = self.parse_assign_target()281            target.set_ctx("param")282            targets.append(target)283            self.stream.expect("assign")284            values.append(self.parse_expression())285        node.targets = targets286        node.values = values287        node.body = self.parse_statements(("name:endwith",), drop_needle=True)288        return node289 290    def parse_autoescape(self) -> nodes.Scope:291        node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)292        node.options = [nodes.Keyword("autoescape", self.parse_expression())]293        node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)294        return nodes.Scope([node])295 296    def parse_block(self) -> nodes.Block:297        node = nodes.Block(lineno=next(self.stream).lineno)298        node.name = self.stream.expect("name").value299        node.scoped = self.stream.skip_if("name:scoped")300        node.required = self.stream.skip_if("name:required")301 302        # common problem people encounter when switching from django303        # to jinja.  we do not support hyphens in block names, so let's304        # raise a nicer error message in that case.305        if self.stream.current.type == "sub":306            self.fail(307                "Block names in Jinja have to be valid Python identifiers and may not"308                " contain hyphens, use an underscore instead."309            )310 311        node.body = self.parse_statements(("name:endblock",), drop_needle=True)312 313        # enforce that required blocks only contain whitespace or comments314        # by asserting that the body, if not empty, is just TemplateData nodes315        # with whitespace data316        if node.required:317            for body_node in node.body:318                if not isinstance(body_node, nodes.Output) or any(319                    not isinstance(output_node, nodes.TemplateData)320                    or not output_node.data.isspace()321                    for output_node in body_node.nodes322                ):323                    self.fail("Required blocks can only contain comments or whitespace")324 325        self.stream.skip_if("name:" + node.name)326        return node327 328    def parse_extends(self) -> nodes.Extends:329        node = nodes.Extends(lineno=next(self.stream).lineno)330        node.template = self.parse_expression()331        return node332 333    def parse_import_context(334        self, node: _ImportInclude, default: bool335    ) -> _ImportInclude:336        if self.stream.current.test_any(337            "name:with", "name:without"338        ) and self.stream.look().test("name:context"):339            node.with_context = next(self.stream).value == "with"340            self.stream.skip()341        else:342            node.with_context = default343        return node344 345    def parse_include(self) -> nodes.Include:346        node = nodes.Include(lineno=next(self.stream).lineno)347        node.template = self.parse_expression()348        if self.stream.current.test("name:ignore") and self.stream.look().test(349            "name:missing"350        ):351            node.ignore_missing = True352            self.stream.skip(2)353        else:354            node.ignore_missing = False355        return self.parse_import_context(node, True)356 357    def parse_import(self) -> nodes.Import:358        node = nodes.Import(lineno=next(self.stream).lineno)359        node.template = self.parse_expression()360        self.stream.expect("name:as")361        node.target = self.parse_assign_target(name_only=True).name362        return self.parse_import_context(node, False)363 364    def parse_from(self) -> nodes.FromImport:365        node = nodes.FromImport(lineno=next(self.stream).lineno)366        node.template = self.parse_expression()367        self.stream.expect("name:import")368        node.names = []369 370        def parse_context() -> bool:371            if self.stream.current.value in {372                "with",373                "without",374            } and self.stream.look().test("name:context"):375                node.with_context = next(self.stream).value == "with"376                self.stream.skip()377                return True378            return False379 380        while True:381            if node.names:382                self.stream.expect("comma")383            if self.stream.current.type == "name":384                if parse_context():385                    break386                target = self.parse_assign_target(name_only=True)387                if target.name.startswith("_"):388                    self.fail(389                        "names starting with an underline can not be imported",390                        target.lineno,391                        exc=TemplateAssertionError,392                    )393                if self.stream.skip_if("name:as"):394                    alias = self.parse_assign_target(name_only=True)395                    node.names.append((target.name, alias.name))396                else:397                    node.names.append(target.name)398                if parse_context() or self.stream.current.type != "comma":399                    break400            else:401                self.stream.expect("name")402        if not hasattr(node, "with_context"):403            node.with_context = False404        return node405 406    def parse_signature(self, node: _MacroCall) -> None:407        args = node.args = []408        defaults = node.defaults = []409        self.stream.expect("lparen")410        while self.stream.current.type != "rparen":411            if args:412                self.stream.expect("comma")413            arg = self.parse_assign_target(name_only=True)414            arg.set_ctx("param")415            if self.stream.skip_if("assign"):416                defaults.append(self.parse_expression())417            elif defaults:418                self.fail("non-default argument follows default argument")419            args.append(arg)420        self.stream.expect("rparen")421 422    def parse_call_block(self) -> nodes.CallBlock:423        node = nodes.CallBlock(lineno=next(self.stream).lineno)424        if self.stream.current.type == "lparen":425            self.parse_signature(node)426        else:427            node.args = []428            node.defaults = []429 430        call_node = self.parse_expression()431        if not isinstance(call_node, nodes.Call):432            self.fail("expected call", node.lineno)433        node.call = call_node434        node.body = self.parse_statements(("name:endcall",), drop_needle=True)435        return node436 437    def parse_filter_block(self) -> nodes.FilterBlock:438        node = nodes.FilterBlock(lineno=next(self.stream).lineno)439        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore440        node.body = self.parse_statements(("name:endfilter",), drop_needle=True)441        return node442 443    def parse_macro(self) -> nodes.Macro:444        node = nodes.Macro(lineno=next(self.stream).lineno)445        node.name = self.parse_assign_target(name_only=True).name446        self.parse_signature(node)447        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)448        return node449 450    def parse_print(self) -> nodes.Output:451        node = nodes.Output(lineno=next(self.stream).lineno)452        node.nodes = []453        while self.stream.current.type != "block_end":454            if node.nodes:455                self.stream.expect("comma")456            node.nodes.append(self.parse_expression())457        return node458 459    @typing.overload460    def parse_assign_target(461        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...462    ) -> nodes.Name: ...463 464    @typing.overload465    def parse_assign_target(466        self,467        with_tuple: bool = True,468        name_only: bool = False,469        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,470        with_namespace: bool = False,471    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...472 473    def parse_assign_target(474        self,475        with_tuple: bool = True,476        name_only: bool = False,477        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,478        with_namespace: bool = False,479    ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:480        """Parse an assignment target.  As Jinja allows assignments to481        tuples, this function can parse all allowed assignment targets.  Per482        default assignments to tuples are parsed, that can be disable however483        by setting `with_tuple` to `False`.  If only assignments to names are484        wanted `name_only` can be set to `True`.  The `extra_end_rules`485        parameter is forwarded to the tuple parsing function.  If486        `with_namespace` is enabled, a namespace assignment may be parsed.487        """488        target: nodes.Expr489 490        if name_only:491            token = self.stream.expect("name")492            target = nodes.Name(token.value, "store", lineno=token.lineno)493        else:494            if with_tuple:495                target = self.parse_tuple(496                    simplified=True,497                    extra_end_rules=extra_end_rules,498                    with_namespace=with_namespace,499                )500            else:501                target = self.parse_primary(with_namespace=with_namespace)502 503            target.set_ctx("store")504 505        if not target.can_assign():506            self.fail(507                f"can't assign to {type(target).__name__.lower()!r}", target.lineno508            )509 510        return target  # type: ignore511 512    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:513        """Parse an expression.  Per default all expressions are parsed, if514        the optional `with_condexpr` parameter is set to `False` conditional515        expressions are not parsed.516        """517        if with_condexpr:518            return self.parse_condexpr()519        return self.parse_or()520 521    def parse_condexpr(self) -> nodes.Expr:522        lineno = self.stream.current.lineno523        expr1 = self.parse_or()524        expr3: t.Optional[nodes.Expr]525 526        while self.stream.skip_if("name:if"):527            expr2 = self.parse_or()528            if self.stream.skip_if("name:else"):529                expr3 = self.parse_condexpr()530            else:531                expr3 = None532            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)533            lineno = self.stream.current.lineno534        return expr1535 536    def parse_or(self) -> nodes.Expr:537        lineno = self.stream.current.lineno538        left = self.parse_and()539        while self.stream.skip_if("name:or"):540            right = self.parse_and()541            left = nodes.Or(left, right, lineno=lineno)542            lineno = self.stream.current.lineno543        return left544 545    def parse_and(self) -> nodes.Expr:546        lineno = self.stream.current.lineno547        left = self.parse_not()548        while self.stream.skip_if("name:and"):549            right = self.parse_not()550            left = nodes.And(left, right, lineno=lineno)551            lineno = self.stream.current.lineno552        return left553 554    def parse_not(self) -> nodes.Expr:555        if self.stream.current.test("name:not"):556            lineno = next(self.stream).lineno557            return nodes.Not(self.parse_not(), lineno=lineno)558        return self.parse_compare()559 560    def parse_compare(self) -> nodes.Expr:561        lineno = self.stream.current.lineno562        expr = self.parse_math1()563        ops = []564        while True:565            token_type = self.stream.current.type566            if token_type in _compare_operators:567                next(self.stream)568                ops.append(nodes.Operand(token_type, self.parse_math1()))569            elif self.stream.skip_if("name:in"):570                ops.append(nodes.Operand("in", self.parse_math1()))571            elif self.stream.current.test("name:not") and self.stream.look().test(572                "name:in"573            ):574                self.stream.skip(2)575                ops.append(nodes.Operand("notin", self.parse_math1()))576            else:577                break578            lineno = self.stream.current.lineno579        if not ops:580            return expr581        return nodes.Compare(expr, ops, lineno=lineno)582 583    def parse_math1(self) -> nodes.Expr:584        lineno = self.stream.current.lineno585        left = self.parse_concat()586        while self.stream.current.type in ("add", "sub"):587            cls = _math_nodes[self.stream.current.type]588            next(self.stream)589            right = self.parse_concat()590            left = cls(left, right, lineno=lineno)591            lineno = self.stream.current.lineno592        return left593 594    def parse_concat(self) -> nodes.Expr:595        lineno = self.stream.current.lineno596        args = [self.parse_math2()]597        while self.stream.current.type == "tilde":598            next(self.stream)599            args.append(self.parse_math2())600        if len(args) == 1:601            return args[0]602        return nodes.Concat(args, lineno=lineno)603 604    def parse_math2(self) -> nodes.Expr:605        lineno = self.stream.current.lineno606        left = self.parse_pow()607        while self.stream.current.type in ("mul", "div", "floordiv", "mod"):608            cls = _math_nodes[self.stream.current.type]609            next(self.stream)610            right = self.parse_pow()611            left = cls(left, right, lineno=lineno)612            lineno = self.stream.current.lineno613        return left614 615    def parse_pow(self) -> nodes.Expr:616        lineno = self.stream.current.lineno617        left = self.parse_unary()618        while self.stream.current.type == "pow":619            next(self.stream)620            right = self.parse_unary()621            left = nodes.Pow(left, right, lineno=lineno)622            lineno = self.stream.current.lineno623        return left624 625    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:626        token_type = self.stream.current.type627        lineno = self.stream.current.lineno628        node: nodes.Expr629 630        if token_type == "sub":631            next(self.stream)632            node = nodes.Neg(self.parse_unary(False), lineno=lineno)633        elif token_type == "add":634            next(self.stream)635            node = nodes.Pos(self.parse_unary(False), lineno=lineno)636        else:637            node = self.parse_primary()638        node = self.parse_postfix(node)639        if with_filter:640            node = self.parse_filter_expr(node)641        return node642 643    def parse_primary(self, with_namespace: bool = False) -> nodes.Expr:644        """Parse a name or literal value. If ``with_namespace`` is enabled, also645        parse namespace attr refs, for use in assignments."""646        token = self.stream.current647        node: nodes.Expr648        if token.type == "name":649            next(self.stream)650            if token.value in ("true", "false", "True", "False"):651                node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)652            elif token.value in ("none", "None"):653                node = nodes.Const(None, lineno=token.lineno)654            elif with_namespace and self.stream.current.type == "dot":655                # If namespace attributes are allowed at this point, and the next656                # token is a dot, produce a namespace reference.657                next(self.stream)658                attr = self.stream.expect("name")659                node = nodes.NSRef(token.value, attr.value, lineno=token.lineno)660            else:661                node = nodes.Name(token.value, "load", lineno=token.lineno)662        elif token.type == "string":663            next(self.stream)664            buf = [token.value]665            lineno = token.lineno666            while self.stream.current.type == "string":667                buf.append(self.stream.current.value)668                next(self.stream)669            node = nodes.Const("".join(buf), lineno=lineno)670        elif token.type in ("integer", "float"):671            next(self.stream)672            node = nodes.Const(token.value, lineno=token.lineno)673        elif token.type == "lparen":674            next(self.stream)675            node = self.parse_tuple(explicit_parentheses=True)676            self.stream.expect("rparen")677        elif token.type == "lbracket":678            node = self.parse_list()679        elif token.type == "lbrace":680            node = self.parse_dict()681        else:682            self.fail(f"unexpected {describe_token(token)!r}", token.lineno)683        return node684 685    def parse_tuple(686        self,687        simplified: bool = False,688        with_condexpr: bool = True,689        extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,690        explicit_parentheses: bool = False,691        with_namespace: bool = False,692    ) -> t.Union[nodes.Tuple, nodes.Expr]:693        """Works like `parse_expression` but if multiple expressions are694        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.695        This method could also return a regular expression instead of a tuple696        if no commas where found.697 698        The default parsing mode is a full tuple.  If `simplified` is `True`699        only names and literals are parsed; ``with_namespace`` allows namespace700        attr refs as well. The `no_condexpr` parameter is forwarded to701        :meth:`parse_expression`.702 703        Because tuples do not require delimiters and may end in a bogus comma704        an extra hint is needed that marks the end of a tuple.  For example705        for loops support tuples between `for` and `in`.  In that case the706        `extra_end_rules` is set to ``['name:in']``.707 708        `explicit_parentheses` is true if the parsing was triggered by an709        expression in parentheses.  This is used to figure out if an empty710        tuple is a valid expression or not.711        """712        lineno = self.stream.current.lineno713        if simplified:714 715            def parse() -> nodes.Expr:716                return self.parse_primary(with_namespace=with_namespace)717 718        else:719 720            def parse() -> nodes.Expr:721                return self.parse_expression(with_condexpr=with_condexpr)722 723        args: t.List[nodes.Expr] = []724        is_tuple = False725 726        while True:727            if args:728                self.stream.expect("comma")729            if self.is_tuple_end(extra_end_rules):730                break731            args.append(parse())732            if self.stream.current.type == "comma":733                is_tuple = True734            else:735                break736            lineno = self.stream.current.lineno737 738        if not is_tuple:739            if args:740                return args[0]741 742            # if we don't have explicit parentheses, an empty tuple is743            # not a valid expression.  This would mean nothing (literally744            # nothing) in the spot of an expression would be an empty745            # tuple.746            if not explicit_parentheses:747                self.fail(748                    "Expected an expression,"749                    f" got {describe_token(self.stream.current)!r}"750                )751 752        return nodes.Tuple(args, "load", lineno=lineno)753 754    def parse_list(self) -> nodes.List:755        token = self.stream.expect("lbracket")756        items: t.List[nodes.Expr] = []757        while self.stream.current.type != "rbracket":758            if items:759                self.stream.expect("comma")760            if self.stream.current.type == "rbracket":761                break762            items.append(self.parse_expression())763        self.stream.expect("rbracket")764        return nodes.List(items, lineno=token.lineno)765 766    def parse_dict(self) -> nodes.Dict:767        token = self.stream.expect("lbrace")768        items: t.List[nodes.Pair] = []769        while self.stream.current.type != "rbrace":770            if items:771                self.stream.expect("comma")772            if self.stream.current.type == "rbrace":773                break774            key = self.parse_expression()775            self.stream.expect("colon")776            value = self.parse_expression()777            items.append(nodes.Pair(key, value, lineno=key.lineno))778        self.stream.expect("rbrace")779        return nodes.Dict(items, lineno=token.lineno)780 781    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:782        while True:783            token_type = self.stream.current.type784            if token_type == "dot" or token_type == "lbracket":785                node = self.parse_subscript(node)786            # calls are valid both after postfix expressions (getattr787            # and getitem) as well as filters and tests788            elif token_type == "lparen":789                node = self.parse_call(node)790            else:791                break792        return node793 794    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:795        while True:796            token_type = self.stream.current.type797            if token_type == "pipe":798                node = self.parse_filter(node)  # type: ignore799            elif token_type == "name" and self.stream.current.value == "is":800                node = self.parse_test(node)801            # calls are valid both after postfix expressions (getattr802            # and getitem) as well as filters and tests803            elif token_type == "lparen":804                node = self.parse_call(node)805            else:806                break807        return node808 809    def parse_subscript(810        self, node: nodes.Expr811    ) -> t.Union[nodes.Getattr, nodes.Getitem]:812        token = next(self.stream)813        arg: nodes.Expr814 815        if token.type == "dot":816            attr_token = self.stream.current817            next(self.stream)818            if attr_token.type == "name":819                return nodes.Getattr(820                    node, attr_token.value, "load", lineno=token.lineno821                )822            elif attr_token.type != "integer":823                self.fail("expected name or number", attr_token.lineno)824            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)825            return nodes.Getitem(node, arg, "load", lineno=token.lineno)826        if token.type == "lbracket":827            args: t.List[nodes.Expr] = []828            while self.stream.current.type != "rbracket":829                if args:830                    self.stream.expect("comma")831                args.append(self.parse_subscribed())832            self.stream.expect("rbracket")833            if len(args) == 1:834                arg = args[0]835            else:836                arg = nodes.Tuple(args, "load", lineno=token.lineno)837            return nodes.Getitem(node, arg, "load", lineno=token.lineno)838        self.fail("expected subscript expression", token.lineno)839 840    def parse_subscribed(self) -> nodes.Expr:841        lineno = self.stream.current.lineno842        args: t.List[t.Optional[nodes.Expr]]843 844        if self.stream.current.type == "colon":845            next(self.stream)846            args = [None]847        else:848            node = self.parse_expression()849            if self.stream.current.type != "colon":850                return node851            next(self.stream)852            args = [node]853 854        if self.stream.current.type == "colon":855            args.append(None)856        elif self.stream.current.type not in ("rbracket", "comma"):857            args.append(self.parse_expression())858        else:859            args.append(None)860 861        if self.stream.current.type == "colon":862            next(self.stream)863            if self.stream.current.type not in ("rbracket", "comma"):864                args.append(self.parse_expression())865            else:866                args.append(None)867        else:868            args.append(None)869 870        return nodes.Slice(lineno=lineno, *args)  # noqa: B026871 872    def parse_call_args(873        self,874    ) -> t.Tuple[875        t.List[nodes.Expr],876        t.List[nodes.Keyword],877        t.Optional[nodes.Expr],878        t.Optional[nodes.Expr],879    ]:880        token = self.stream.expect("lparen")881        args = []882        kwargs = []883        dyn_args = None884        dyn_kwargs = None885        require_comma = False886 887        def ensure(expr: bool) -> None:888            if not expr:889                self.fail("invalid syntax for function call expression", token.lineno)890 891        while self.stream.current.type != "rparen":892            if require_comma:893                self.stream.expect("comma")894 895                # support for trailing comma896                if self.stream.current.type == "rparen":897                    break898 899            if self.stream.current.type == "mul":900                ensure(dyn_args is None and dyn_kwargs is None)901                next(self.stream)902                dyn_args = self.parse_expression()903            elif self.stream.current.type == "pow":904                ensure(dyn_kwargs is None)905                next(self.stream)906                dyn_kwargs = self.parse_expression()907            else:908                if (909                    self.stream.current.type == "name"910                    and self.stream.look().type == "assign"911                ):912                    # Parsing a kwarg913                    ensure(dyn_kwargs is None)914                    key = self.stream.current.value915                    self.stream.skip(2)916                    value = self.parse_expression()917                    kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))918                else:919                    # Parsing an arg920                    ensure(dyn_args is None and dyn_kwargs is None and not kwargs)921                    args.append(self.parse_expression())922 923            require_comma = True924 925        self.stream.expect("rparen")926        return args, kwargs, dyn_args, dyn_kwargs927 928    def parse_call(self, node: nodes.Expr) -> nodes.Call:929        # The lparen will be expected in parse_call_args, but the lineno930        # needs to be recorded before the stream is advanced.931        token = self.stream.current932        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()933        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)934 935    def parse_filter(936        self, node: t.Optional[nodes.Expr], start_inline: bool = False937    ) -> t.Optional[nodes.Expr]:938        while self.stream.current.type == "pipe" or start_inline:939            if not start_inline:940                next(self.stream)941            token = self.stream.expect("name")942            name = token.value943            while self.stream.current.type == "dot":944                next(self.stream)945                name += "." + self.stream.expect("name").value946            if self.stream.current.type == "lparen":947                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()948            else:949                args = []950                kwargs = []951                dyn_args = dyn_kwargs = None952            node = nodes.Filter(953                node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno954            )955            start_inline = False956        return node957 958    def parse_test(self, node: nodes.Expr) -> nodes.Expr:959        token = next(self.stream)960        if self.stream.current.test("name:not"):961            next(self.stream)962            negated = True963        else:964            negated = False965        name = self.stream.expect("name").value966        while self.stream.current.type == "dot":967            next(self.stream)968            name += "." + self.stream.expect("name").value969        dyn_args = dyn_kwargs = None970        kwargs: t.List[nodes.Keyword] = []971        if self.stream.current.type == "lparen":972            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()973        elif self.stream.current.type in {974            "name",975            "string",976            "integer",977            "float",978            "lparen",979            "lbracket",980            "lbrace",981        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):982            if self.stream.current.test("name:is"):983                self.fail("You cannot chain multiple tests with is")984            arg_node = self.parse_primary()985            arg_node = self.parse_postfix(arg_node)986            args = [arg_node]987        else:988            args = []989        node = nodes.Test(990            node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno991        )992        if negated:993            node = nodes.Not(node, lineno=token.lineno)994        return node995 996    def subparse(997        self, end_tokens: t.Optional[t.Tuple[str, ...]] = None998    ) -> t.List[nodes.Node]:999        body: t.List[nodes.Node] = []1000        data_buffer: t.List[nodes.Node] = []1001        add_data = data_buffer.append1002 1003        if end_tokens is not None:1004            self._end_token_stack.append(end_tokens)1005 1006        def flush_data() -> None:1007            if data_buffer:1008                lineno = data_buffer[0].lineno1009                body.append(nodes.Output(data_buffer[:], lineno=lineno))1010                del data_buffer[:]1011 1012        try:1013            while self.stream:1014                token = self.stream.current1015                if token.type == "data":1016                    if token.value:1017                        add_data(nodes.TemplateData(token.value, lineno=token.lineno))1018                    next(self.stream)1019                elif token.type == "variable_begin":1020                    next(self.stream)1021                    add_data(self.parse_tuple(with_condexpr=True))1022                    self.stream.expect("variable_end")1023                elif token.type == "block_begin":1024                    flush_data()1025                    next(self.stream)1026                    if end_tokens is not None and self.stream.current.test_any(1027                        *end_tokens1028                    ):1029                        return body1030                    rv = self.parse_statement()1031                    if isinstance(rv, list):1032                        body.extend(rv)1033                    else:1034                        body.append(rv)1035                    self.stream.expect("block_end")1036                else:1037                    raise AssertionError("internal parsing error")1038 1039            flush_data()1040        finally:1041            if end_tokens is not None:1042                self._end_token_stack.pop()1043        return body1044 1045    def parse(self) -> nodes.Template:1046        """Parse the whole template into a `Template` node."""1047        result = nodes.Template(self.subparse(), lineno=1)1048        result.set_environment(self.environment)1049        return result1050