codekingpro/portable-devtools

1"""Extension API for adding custom tags and behavior."""2 3import pprint4import re5import typing as t6 7from markupsafe import Markup8 9from . import defaults10from . import nodes11from .environment import Environment12from .exceptions import TemplateAssertionError13from .exceptions import TemplateSyntaxError14from .runtime import concat  # type: ignore15from .runtime import Context16from .runtime import Undefined17from .utils import import_string18from .utils import pass_context19 20if t.TYPE_CHECKING:21    import typing_extensions as te22 23    from .lexer import Token24    from .lexer import TokenStream25    from .parser import Parser26 27    class _TranslationsBasic(te.Protocol):28        def gettext(self, message: str) -> str: ...29 30        def ngettext(self, singular: str, plural: str, n: int) -> str:31            pass32 33    class _TranslationsContext(_TranslationsBasic):34        def pgettext(self, context: str, message: str) -> str: ...35 36        def npgettext(37            self, context: str, singular: str, plural: str, n: int38        ) -> str: ...39 40    _SupportedTranslations = t.Union[_TranslationsBasic, _TranslationsContext]41 42 43# I18N functions available in Jinja templates. If the I18N library44# provides ugettext, it will be assigned to gettext.45GETTEXT_FUNCTIONS: t.Tuple[str, ...] = (46    "_",47    "gettext",48    "ngettext",49    "pgettext",50    "npgettext",51)52_ws_re = re.compile(r"\s*\n\s*")53 54 55class Extension:56    """Extensions can be used to add extra functionality to the Jinja template57    system at the parser level.  Custom extensions are bound to an environment58    but may not store environment specific data on `self`.  The reason for59    this is that an extension can be bound to another environment (for60    overlays) by creating a copy and reassigning the `environment` attribute.61 62    As extensions are created by the environment they cannot accept any63    arguments for configuration.  One may want to work around that by using64    a factory function, but that is not possible as extensions are identified65    by their import name.  The correct way to configure the extension is66    storing the configuration values on the environment.  Because this way the67    environment ends up acting as central configuration storage the68    attributes may clash which is why extensions have to ensure that the names69    they choose for configuration are not too generic.  ``prefix`` for example70    is a terrible name, ``fragment_cache_prefix`` on the other hand is a good71    name as includes the name of the extension (fragment cache).72    """73 74    identifier: t.ClassVar[str]75 76    def __init_subclass__(cls) -> None:77        cls.identifier = f"{cls.__module__}.{cls.__name__}"78 79    #: if this extension parses this is the list of tags it's listening to.80    tags: t.Set[str] = set()81 82    #: the priority of that extension.  This is especially useful for83    #: extensions that preprocess values.  A lower value means higher84    #: priority.85    #:86    #: .. versionadded:: 2.487    priority = 10088 89    def __init__(self, environment: Environment) -> None:90        self.environment = environment91 92    def bind(self, environment: Environment) -> "te.Self":93        """Create a copy of this extension bound to another environment."""94        rv = object.__new__(self.__class__)95        rv.__dict__.update(self.__dict__)96        rv.environment = environment97        return rv98 99    def preprocess(100        self, source: str, name: t.Optional[str], filename: t.Optional[str] = None101    ) -> str:102        """This method is called before the actual lexing and can be used to103        preprocess the source.  The `filename` is optional.  The return value104        must be the preprocessed source.105        """106        return source107 108    def filter_stream(109        self, stream: "TokenStream"110    ) -> t.Union["TokenStream", t.Iterable["Token"]]:111        """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used112        to filter tokens returned.  This method has to return an iterable of113        :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a114        :class:`~jinja2.lexer.TokenStream`.115        """116        return stream117 118    def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]:119        """If any of the :attr:`tags` matched this method is called with the120        parser as first argument.  The token the parser stream is pointing at121        is the name token that matched.  This method has to return one or a122        list of multiple nodes.123        """124        raise NotImplementedError()125 126    def attr(127        self, name: str, lineno: t.Optional[int] = None128    ) -> nodes.ExtensionAttribute:129        """Return an attribute node for the current extension.  This is useful130        to pass constants on extensions to generated template code.131 132        ::133 134            self.attr('_my_attribute', lineno=lineno)135        """136        return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)137 138    def call_method(139        self,140        name: str,141        args: t.Optional[t.List[nodes.Expr]] = None,142        kwargs: t.Optional[t.List[nodes.Keyword]] = None,143        dyn_args: t.Optional[nodes.Expr] = None,144        dyn_kwargs: t.Optional[nodes.Expr] = None,145        lineno: t.Optional[int] = None,146    ) -> nodes.Call:147        """Call a method of the extension.  This is a shortcut for148        :meth:`attr` + :class:`jinja2.nodes.Call`.149        """150        if args is None:151            args = []152        if kwargs is None:153            kwargs = []154        return nodes.Call(155            self.attr(name, lineno=lineno),156            args,157            kwargs,158            dyn_args,159            dyn_kwargs,160            lineno=lineno,161        )162 163 164@pass_context165def _gettext_alias(166    __context: Context, *args: t.Any, **kwargs: t.Any167) -> t.Union[t.Any, Undefined]:168    return __context.call(__context.resolve("gettext"), *args, **kwargs)169 170 171def _make_new_gettext(func: t.Callable[[str], str]) -> t.Callable[..., str]:172    @pass_context173    def gettext(__context: Context, __string: str, **variables: t.Any) -> str:174        rv = __context.call(func, __string)175        if __context.eval_ctx.autoescape:176            rv = Markup(rv)177        # Always treat as a format string, even if there are no178        # variables. This makes translation strings more consistent179        # and predictable. This requires escaping180        return rv % variables  # type: ignore181 182    return gettext183 184 185def _make_new_ngettext(func: t.Callable[[str, str, int], str]) -> t.Callable[..., str]:186    @pass_context187    def ngettext(188        __context: Context,189        __singular: str,190        __plural: str,191        __num: int,192        **variables: t.Any,193    ) -> str:194        variables.setdefault("num", __num)195        rv = __context.call(func, __singular, __plural, __num)196        if __context.eval_ctx.autoescape:197            rv = Markup(rv)198        # Always treat as a format string, see gettext comment above.199        return rv % variables  # type: ignore200 201    return ngettext202 203 204def _make_new_pgettext(func: t.Callable[[str, str], str]) -> t.Callable[..., str]:205    @pass_context206    def pgettext(207        __context: Context, __string_ctx: str, __string: str, **variables: t.Any208    ) -> str:209        variables.setdefault("context", __string_ctx)210        rv = __context.call(func, __string_ctx, __string)211 212        if __context.eval_ctx.autoescape:213            rv = Markup(rv)214 215        # Always treat as a format string, see gettext comment above.216        return rv % variables  # type: ignore217 218    return pgettext219 220 221def _make_new_npgettext(222    func: t.Callable[[str, str, str, int], str],223) -> t.Callable[..., str]:224    @pass_context225    def npgettext(226        __context: Context,227        __string_ctx: str,228        __singular: str,229        __plural: str,230        __num: int,231        **variables: t.Any,232    ) -> str:233        variables.setdefault("context", __string_ctx)234        variables.setdefault("num", __num)235        rv = __context.call(func, __string_ctx, __singular, __plural, __num)236 237        if __context.eval_ctx.autoescape:238            rv = Markup(rv)239 240        # Always treat as a format string, see gettext comment above.241        return rv % variables  # type: ignore242 243    return npgettext244 245 246class InternationalizationExtension(Extension):247    """This extension adds gettext support to Jinja."""248 249    tags = {"trans"}250 251    # TODO: the i18n extension is currently reevaluating values in a few252    # situations.  Take this example:253    #   {% trans count=something() %}{{ count }} foo{% pluralize254    #     %}{{ count }} fooss{% endtrans %}255    # something is called twice here.  One time for the gettext value and256    # the other time for the n-parameter of the ngettext function.257 258    def __init__(self, environment: Environment) -> None:259        super().__init__(environment)260        environment.globals["_"] = _gettext_alias261        environment.extend(262            install_gettext_translations=self._install,263            install_null_translations=self._install_null,264            install_gettext_callables=self._install_callables,265            uninstall_gettext_translations=self._uninstall,266            extract_translations=self._extract,267            newstyle_gettext=False,268        )269 270    def _install(271        self, translations: "_SupportedTranslations", newstyle: t.Optional[bool] = None272    ) -> None:273        # ugettext and ungettext are preferred in case the I18N library274        # is providing compatibility with older Python versions.275        gettext = getattr(translations, "ugettext", None)276        if gettext is None:277            gettext = translations.gettext278        ngettext = getattr(translations, "ungettext", None)279        if ngettext is None:280            ngettext = translations.ngettext281 282        pgettext = getattr(translations, "pgettext", None)283        npgettext = getattr(translations, "npgettext", None)284        self._install_callables(285            gettext, ngettext, newstyle=newstyle, pgettext=pgettext, npgettext=npgettext286        )287 288    def _install_null(self, newstyle: t.Optional[bool] = None) -> None:289        import gettext290 291        translations = gettext.NullTranslations()292 293        if hasattr(translations, "pgettext"):294            # Python < 3.8295            pgettext = translations.pgettext296        else:297 298            def pgettext(c: str, s: str) -> str:  # type: ignore[misc]299                return s300 301        if hasattr(translations, "npgettext"):302            npgettext = translations.npgettext303        else:304 305            def npgettext(c: str, s: str, p: str, n: int) -> str:  # type: ignore[misc]306                return s if n == 1 else p307 308        self._install_callables(309            gettext=translations.gettext,310            ngettext=translations.ngettext,311            newstyle=newstyle,312            pgettext=pgettext,313            npgettext=npgettext,314        )315 316    def _install_callables(317        self,318        gettext: t.Callable[[str], str],319        ngettext: t.Callable[[str, str, int], str],320        newstyle: t.Optional[bool] = None,321        pgettext: t.Optional[t.Callable[[str, str], str]] = None,322        npgettext: t.Optional[t.Callable[[str, str, str, int], str]] = None,323    ) -> None:324        if newstyle is not None:325            self.environment.newstyle_gettext = newstyle  # type: ignore326        if self.environment.newstyle_gettext:  # type: ignore327            gettext = _make_new_gettext(gettext)328            ngettext = _make_new_ngettext(ngettext)329 330            if pgettext is not None:331                pgettext = _make_new_pgettext(pgettext)332 333            if npgettext is not None:334                npgettext = _make_new_npgettext(npgettext)335 336        self.environment.globals.update(337            gettext=gettext, ngettext=ngettext, pgettext=pgettext, npgettext=npgettext338        )339 340    def _uninstall(self, translations: "_SupportedTranslations") -> None:341        for key in ("gettext", "ngettext", "pgettext", "npgettext"):342            self.environment.globals.pop(key, None)343 344    def _extract(345        self,346        source: t.Union[str, nodes.Template],347        gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,348    ) -> t.Iterator[349        t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]]350    ]:351        if isinstance(source, str):352            source = self.environment.parse(source)353        return extract_from_ast(source, gettext_functions)354 355    def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]:356        """Parse a translatable tag."""357        lineno = next(parser.stream).lineno358 359        context = None360        context_token = parser.stream.next_if("string")361 362        if context_token is not None:363            context = context_token.value364 365        # find all the variables referenced.  Additionally a variable can be366        # defined in the body of the trans block too, but this is checked at367        # a later state.368        plural_expr: t.Optional[nodes.Expr] = None369        plural_expr_assignment: t.Optional[nodes.Assign] = None370        num_called_num = False371        variables: t.Dict[str, nodes.Expr] = {}372        trimmed = None373        while parser.stream.current.type != "block_end":374            if variables:375                parser.stream.expect("comma")376 377            # skip colon for python compatibility378            if parser.stream.skip_if("colon"):379                break380 381            token = parser.stream.expect("name")382            if token.value in variables:383                parser.fail(384                    f"translatable variable {token.value!r} defined twice.",385                    token.lineno,386                    exc=TemplateAssertionError,387                )388 389            # expressions390            if parser.stream.current.type == "assign":391                next(parser.stream)392                variables[token.value] = var = parser.parse_expression()393            elif trimmed is None and token.value in ("trimmed", "notrimmed"):394                trimmed = token.value == "trimmed"395                continue396            else:397                variables[token.value] = var = nodes.Name(token.value, "load")398 399            if plural_expr is None:400                if isinstance(var, nodes.Call):401                    plural_expr = nodes.Name("_trans", "load")402                    variables[token.value] = plural_expr403                    plural_expr_assignment = nodes.Assign(404                        nodes.Name("_trans", "store"), var405                    )406                else:407                    plural_expr = var408                num_called_num = token.value == "num"409 410        parser.stream.expect("block_end")411 412        plural = None413        have_plural = False414        referenced = set()415 416        # now parse until endtrans or pluralize417        singular_names, singular = self._parse_block(parser, True)418        if singular_names:419            referenced.update(singular_names)420            if plural_expr is None:421                plural_expr = nodes.Name(singular_names[0], "load")422                num_called_num = singular_names[0] == "num"423 424        # if we have a pluralize block, we parse that too425        if parser.stream.current.test("name:pluralize"):426            have_plural = True427            next(parser.stream)428            if parser.stream.current.type != "block_end":429                token = parser.stream.expect("name")430                if token.value not in variables:431                    parser.fail(432                        f"unknown variable {token.value!r} for pluralization",433                        token.lineno,434                        exc=TemplateAssertionError,435                    )436                plural_expr = variables[token.value]437                num_called_num = token.value == "num"438            parser.stream.expect("block_end")439            plural_names, plural = self._parse_block(parser, False)440            next(parser.stream)441            referenced.update(plural_names)442        else:443            next(parser.stream)444 445        # register free names as simple name expressions446        for name in referenced:447            if name not in variables:448                variables[name] = nodes.Name(name, "load")449 450        if not have_plural:451            plural_expr = None452        elif plural_expr is None:453            parser.fail("pluralize without variables", lineno)454 455        if trimmed is None:456            trimmed = self.environment.policies["ext.i18n.trimmed"]457        if trimmed:458            singular = self._trim_whitespace(singular)459            if plural:460                plural = self._trim_whitespace(plural)461 462        node = self._make_node(463            singular,464            plural,465            context,466            variables,467            plural_expr,468            bool(referenced),469            num_called_num and have_plural,470        )471        node.set_lineno(lineno)472        if plural_expr_assignment is not None:473            return [plural_expr_assignment, node]474        else:475            return node476 477    def _trim_whitespace(self, string: str, _ws_re: t.Pattern[str] = _ws_re) -> str:478        return _ws_re.sub(" ", string.strip())479 480    def _parse_block(481        self, parser: "Parser", allow_pluralize: bool482    ) -> t.Tuple[t.List[str], str]:483        """Parse until the next block tag with a given name."""484        referenced = []485        buf = []486 487        while True:488            if parser.stream.current.type == "data":489                buf.append(parser.stream.current.value.replace("%", "%%"))490                next(parser.stream)491            elif parser.stream.current.type == "variable_begin":492                next(parser.stream)493                name = parser.stream.expect("name").value494                referenced.append(name)495                buf.append(f"%({name})s")496                parser.stream.expect("variable_end")497            elif parser.stream.current.type == "block_begin":498                next(parser.stream)499                block_name = (500                    parser.stream.current.value501                    if parser.stream.current.type == "name"502                    else None503                )504                if block_name == "endtrans":505                    break506                elif block_name == "pluralize":507                    if allow_pluralize:508                        break509                    parser.fail(510                        "a translatable section can have only one pluralize section"511                    )512                elif block_name == "trans":513                    parser.fail(514                        "trans blocks can't be nested; did you mean `endtrans`?"515                    )516                parser.fail(517                    f"control structures in translatable sections are not allowed; "518                    f"saw `{block_name}`"519                )520            elif parser.stream.eos:521                parser.fail("unclosed translation block")522            else:523                raise RuntimeError("internal parser error")524 525        return referenced, concat(buf)526 527    def _make_node(528        self,529        singular: str,530        plural: t.Optional[str],531        context: t.Optional[str],532        variables: t.Dict[str, nodes.Expr],533        plural_expr: t.Optional[nodes.Expr],534        vars_referenced: bool,535        num_called_num: bool,536    ) -> nodes.Output:537        """Generates a useful node from the data provided."""538        newstyle = self.environment.newstyle_gettext  # type: ignore539        node: nodes.Expr540 541        # no variables referenced?  no need to escape for old style542        # gettext invocations only if there are vars.543        if not vars_referenced and not newstyle:544            singular = singular.replace("%%", "%")545            if plural:546                plural = plural.replace("%%", "%")547 548        func_name = "gettext"549        func_args: t.List[nodes.Expr] = [nodes.Const(singular)]550 551        if context is not None:552            func_args.insert(0, nodes.Const(context))553            func_name = f"p{func_name}"554 555        if plural_expr is not None:556            func_name = f"n{func_name}"557            func_args.extend((nodes.Const(plural), plural_expr))558 559        node = nodes.Call(nodes.Name(func_name, "load"), func_args, [], None, None)560 561        # in case newstyle gettext is used, the method is powerful562        # enough to handle the variable expansion and autoescape563        # handling itself564        if newstyle:565            for key, value in variables.items():566                # the function adds that later anyways in case num was567                # called num, so just skip it.568                if num_called_num and key == "num":569                    continue570                node.kwargs.append(nodes.Keyword(key, value))571 572        # otherwise do that here573        else:574            # mark the return value as safe if we are in an575            # environment with autoescaping turned on576            node = nodes.MarkSafeIfAutoescape(node)577            if variables:578                node = nodes.Mod(579                    node,580                    nodes.Dict(581                        [582                            nodes.Pair(nodes.Const(key), value)583                            for key, value in variables.items()584                        ]585                    ),586                )587        return nodes.Output([node])588 589 590class ExprStmtExtension(Extension):591    """Adds a `do` tag to Jinja that works like the print statement just592    that it doesn't print the return value.593    """594 595    tags = {"do"}596 597    def parse(self, parser: "Parser") -> nodes.ExprStmt:598        node = nodes.ExprStmt(lineno=next(parser.stream).lineno)599        node.node = parser.parse_tuple()600        return node601 602 603class LoopControlExtension(Extension):604    """Adds break and continue to the template engine."""605 606    tags = {"break", "continue"}607 608    def parse(self, parser: "Parser") -> t.Union[nodes.Break, nodes.Continue]:609        token = next(parser.stream)610        if token.value == "break":611            return nodes.Break(lineno=token.lineno)612        return nodes.Continue(lineno=token.lineno)613 614 615class DebugExtension(Extension):616    """A ``{% debug %}`` tag that dumps the available variables,617    filters, and tests.618 619    .. code-block:: html+jinja620 621        <pre>{% debug %}</pre>622 623    .. code-block:: text624 625        {'context': {'cycler': <class 'jinja2.utils.Cycler'>,626                     ...,627                     'namespace': <class 'jinja2.utils.Namespace'>},628         'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd',629                     ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'],630         'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined',631                   ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']}632 633    .. versionadded:: 2.11.0634    """635 636    tags = {"debug"}637 638    def parse(self, parser: "Parser") -> nodes.Output:639        lineno = parser.stream.expect("name:debug").lineno640        context = nodes.ContextReference()641        result = self.call_method("_render", [context], lineno=lineno)642        return nodes.Output([result], lineno=lineno)643 644    def _render(self, context: Context) -> str:645        result = {646            "context": context.get_all(),647            "filters": sorted(self.environment.filters.keys()),648            "tests": sorted(self.environment.tests.keys()),649        }650 651        # Set the depth since the intent is to show the top few names.652        return pprint.pformat(result, depth=3, compact=True)653 654 655def extract_from_ast(656    ast: nodes.Template,657    gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,658    babel_style: bool = True,659) -> t.Iterator[660    t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]]661]:662    """Extract localizable strings from the given template node.  Per663    default this function returns matches in babel style that means non string664    parameters as well as keyword arguments are returned as `None`.  This665    allows Babel to figure out what you really meant if you are using666    gettext functions that allow keyword arguments for placeholder expansion.667    If you don't want that behavior set the `babel_style` parameter to `False`668    which causes only strings to be returned and parameters are always stored669    in tuples.  As a consequence invalid gettext calls (calls without a single670    string parameter or string parameters after non-string parameters) are671    skipped.672 673    This example explains the behavior:674 675    >>> from jinja2 import Environment676    >>> env = Environment()677    >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')678    >>> list(extract_from_ast(node))679    [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]680    >>> list(extract_from_ast(node, babel_style=False))681    [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]682 683    For every string found this function yields a ``(lineno, function,684    message)`` tuple, where:685 686    * ``lineno`` is the number of the line on which the string was found,687    * ``function`` is the name of the ``gettext`` function used (if the688      string was extracted from embedded Python code), and689    *   ``message`` is the string, or a tuple of strings for functions690         with multiple string arguments.691 692    This extraction function operates on the AST and is because of that unable693    to extract any comments.  For comment support you have to use the babel694    extraction interface or extract comments yourself.695    """696    out: t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]697 698    for node in ast.find_all(nodes.Call):699        if (700            not isinstance(node.node, nodes.Name)701            or node.node.name not in gettext_functions702        ):703            continue704 705        strings: t.List[t.Optional[str]] = []706 707        for arg in node.args:708            if isinstance(arg, nodes.Const) and isinstance(arg.value, str):709                strings.append(arg.value)710            else:711                strings.append(None)712 713        for _ in node.kwargs:714            strings.append(None)715        if node.dyn_args is not None:716            strings.append(None)717        if node.dyn_kwargs is not None:718            strings.append(None)719 720        if not babel_style:721            out = tuple(x for x in strings if x is not None)722 723            if not out:724                continue725        else:726            if len(strings) == 1:727                out = strings[0]728            else:729                out = tuple(strings)730 731        yield node.lineno, node.node.name, out732 733 734class _CommentFinder:735    """Helper class to find comments in a token stream.  Can only736    find comments for gettext calls forwards.  Once the comment737    from line 4 is found, a comment for line 1 will not return a738    usable value.739    """740 741    def __init__(742        self, tokens: t.Sequence[t.Tuple[int, str, str]], comment_tags: t.Sequence[str]743    ) -> None:744        self.tokens = tokens745        self.comment_tags = comment_tags746        self.offset = 0747        self.last_lineno = 0748 749    def find_backwards(self, offset: int) -> t.List[str]:750        try:751            for _, token_type, token_value in reversed(752                self.tokens[self.offset : offset]753            ):754                if token_type in ("comment", "linecomment"):755                    try:756                        prefix, comment = token_value.split(None, 1)757                    except ValueError:758                        continue759                    if prefix in self.comment_tags:760                        return [comment.rstrip()]761            return []762        finally:763            self.offset = offset764 765    def find_comments(self, lineno: int) -> t.List[str]:766        if not self.comment_tags or self.last_lineno > lineno:767            return []768        for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]):769            if token_lineno > lineno:770                return self.find_backwards(self.offset + idx)771        return self.find_backwards(len(self.tokens))772 773 774def babel_extract(775    fileobj: t.BinaryIO,776    keywords: t.Sequence[str],777    comment_tags: t.Sequence[str],778    options: t.Dict[str, t.Any],779) -> t.Iterator[780    t.Tuple[781        int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]], t.List[str]782    ]783]:784    """Babel extraction method for Jinja templates.785 786    .. versionchanged:: 2.3787       Basic support for translation comments was added.  If `comment_tags`788       is now set to a list of keywords for extraction, the extractor will789       try to find the best preceding comment that begins with one of the790       keywords.  For best results, make sure to not have more than one791       gettext call in one line of code and the matching comment in the792       same line or the line before.793 794    .. versionchanged:: 2.5.1795       The `newstyle_gettext` flag can be set to `True` to enable newstyle796       gettext calls.797 798    .. versionchanged:: 2.7799       A `silent` option can now be provided.  If set to `False` template800       syntax errors are propagated instead of being ignored.801 802    :param fileobj: the file-like object the messages should be extracted from803    :param keywords: a list of keywords (i.e. function names) that should be804                     recognized as translation functions805    :param comment_tags: a list of translator tags to search for and include806                         in the results.807    :param options: a dictionary of additional options (optional)808    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.809             (comments will be empty currently)810    """811    extensions: t.Dict[t.Type[Extension], None] = {}812 813    for extension_name in options.get("extensions", "").split(","):814        extension_name = extension_name.strip()815 816        if not extension_name:817            continue818 819        extensions[import_string(extension_name)] = None820 821    if InternationalizationExtension not in extensions:822        extensions[InternationalizationExtension] = None823 824    def getbool(options: t.Mapping[str, str], key: str, default: bool = False) -> bool:825        return options.get(key, str(default)).lower() in {"1", "on", "yes", "true"}826 827    silent = getbool(options, "silent", True)828    environment = Environment(829        options.get("block_start_string", defaults.BLOCK_START_STRING),830        options.get("block_end_string", defaults.BLOCK_END_STRING),831        options.get("variable_start_string", defaults.VARIABLE_START_STRING),832        options.get("variable_end_string", defaults.VARIABLE_END_STRING),833        options.get("comment_start_string", defaults.COMMENT_START_STRING),834        options.get("comment_end_string", defaults.COMMENT_END_STRING),835        options.get("line_statement_prefix") or defaults.LINE_STATEMENT_PREFIX,836        options.get("line_comment_prefix") or defaults.LINE_COMMENT_PREFIX,837        getbool(options, "trim_blocks", defaults.TRIM_BLOCKS),838        getbool(options, "lstrip_blocks", defaults.LSTRIP_BLOCKS),839        defaults.NEWLINE_SEQUENCE,840        getbool(options, "keep_trailing_newline", defaults.KEEP_TRAILING_NEWLINE),841        tuple(extensions),842        cache_size=0,843        auto_reload=False,844    )845 846    if getbool(options, "trimmed"):847        environment.policies["ext.i18n.trimmed"] = True848    if getbool(options, "newstyle_gettext"):849        environment.newstyle_gettext = True  # type: ignore850 851    source = fileobj.read().decode(options.get("encoding", "utf-8"))852    try:853        node = environment.parse(source)854        tokens = list(environment.lex(environment.preprocess(source)))855    except TemplateSyntaxError:856        if not silent:857            raise858        # skip templates with syntax errors859        return860 861    finder = _CommentFinder(tokens, comment_tags)862    for lineno, func, message in extract_from_ast(node, keywords):863        yield lineno, func, message, finder.find_comments(lineno)864 865 866#: nicer import names867i18n = InternationalizationExtension868do = ExprStmtExtension869loopcontrols = LoopControlExtension870debug = DebugExtension871