codekingpro/portable-devtools

1"""2    babel.messages.extract3    ~~~~~~~~~~~~~~~~~~~~~~4 5    Basic infrastructure for extracting localizable messages from source files.6 7    This module defines an extensible system for collecting localizable message8    strings from a variety of sources. A native extractor for Python source9    files is builtin, extractors for other sources can be added using very10    simple plugins.11 12    The main entry points into the extraction functionality are the functions13    `extract_from_dir` and `extract_from_file`.14 15    :copyright: (c) 2013-2024 by the Babel Team.16    :license: BSD, see LICENSE for more details.17"""18from __future__ import annotations19 20import ast21import io22import os23import sys24import tokenize25from collections.abc import (26    Callable,27    Collection,28    Generator,29    Iterable,30    Mapping,31    MutableSequence,32)33from os.path import relpath34from textwrap import dedent35from tokenize import COMMENT, NAME, OP, STRING, generate_tokens36from typing import TYPE_CHECKING, Any37 38from babel.util import parse_encoding, parse_future_flags, pathmatch39 40if TYPE_CHECKING:41    from typing import IO, Final, Protocol42 43    from _typeshed import SupportsItems, SupportsRead, SupportsReadline44    from typing_extensions import TypeAlias, TypedDict45 46    class _PyOptions(TypedDict, total=False):47        encoding: str48 49    class _JSOptions(TypedDict, total=False):50        encoding: str51        jsx: bool52        template_string: bool53        parse_template_string: bool54 55    class _FileObj(SupportsRead[bytes], SupportsReadline[bytes], Protocol):56        def seek(self, __offset: int, __whence: int = ...) -> int: ...57        def tell(self) -> int: ...58 59    _SimpleKeyword: TypeAlias = tuple[int | tuple[int, int] | tuple[int, str], ...] | None60    _Keyword: TypeAlias = dict[int | None, _SimpleKeyword] | _SimpleKeyword61 62    # 5-tuple of (filename, lineno, messages, comments, context)63    _FileExtractionResult: TypeAlias = tuple[str, int, str | tuple[str, ...], list[str], str | None]64 65    # 4-tuple of (lineno, message, comments, context)66    _ExtractionResult: TypeAlias = tuple[int, str | tuple[str, ...], list[str], str | None]67 68    # Required arguments: fileobj, keywords, comment_tags, options69    # Return value: Iterable of (lineno, message, comments, context)70    _CallableExtractionMethod: TypeAlias = Callable[71        [_FileObj | IO[bytes], Mapping[str, _Keyword], Collection[str], Mapping[str, Any]],72        Iterable[_ExtractionResult],73    ]74 75    _ExtractionMethod: TypeAlias = _CallableExtractionMethod | str76 77GROUP_NAME: Final[str] = 'babel.extractors'78 79DEFAULT_KEYWORDS: dict[str, _Keyword] = {80    '_': None,81    'gettext': None,82    'ngettext': (1, 2),83    'ugettext': None,84    'ungettext': (1, 2),85    'dgettext': (2,),86    'dngettext': (2, 3),87    'N_': None,88    'pgettext': ((1, 'c'), 2),89    'npgettext': ((1, 'c'), 2, 3),90}91 92DEFAULT_MAPPING: list[tuple[str, str]] = [('**.py', 'python')]93 94# New tokens in Python 3.12, or None on older versions95FSTRING_START = getattr(tokenize, "FSTRING_START", None)96FSTRING_MIDDLE = getattr(tokenize, "FSTRING_MIDDLE", None)97FSTRING_END = getattr(tokenize, "FSTRING_END", None)98 99 100def _strip_comment_tags(comments: MutableSequence[str], tags: Iterable[str]):101    """Helper function for `extract` that strips comment tags from strings102    in a list of comment lines.  This functions operates in-place.103    """104    def _strip(line: str):105        for tag in tags:106            if line.startswith(tag):107                return line[len(tag):].strip()108        return line109    comments[:] = map(_strip, comments)110 111 112def default_directory_filter(dirpath: str | os.PathLike[str]) -> bool:113    subdir = os.path.basename(dirpath)114    # Legacy default behavior: ignore dot and underscore directories115    return not (subdir.startswith('.') or subdir.startswith('_'))116 117 118def extract_from_dir(119    dirname: str | os.PathLike[str] | None = None,120    method_map: Iterable[tuple[str, str]] = DEFAULT_MAPPING,121    options_map: SupportsItems[str, dict[str, Any]] | None = None,122    keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,123    comment_tags: Collection[str] = (),124    callback: Callable[[str, str, dict[str, Any]], object] | None = None,125    strip_comment_tags: bool = False,126    directory_filter: Callable[[str], bool] | None = None,127) -> Generator[_FileExtractionResult, None, None]:128    """Extract messages from any source files found in the given directory.129 130    This function generates tuples of the form ``(filename, lineno, message,131    comments, context)``.132 133    Which extraction method is used per file is determined by the `method_map`134    parameter, which maps extended glob patterns to extraction method names.135    For example, the following is the default mapping:136 137    >>> method_map = [138    ...     ('**.py', 'python')139    ... ]140 141    This basically says that files with the filename extension ".py" at any142    level inside the directory should be processed by the "python" extraction143    method. Files that don't match any of the mapping patterns are ignored. See144    the documentation of the `pathmatch` function for details on the pattern145    syntax.146 147    The following extended mapping would also use the "genshi" extraction148    method on any file in "templates" subdirectory:149 150    >>> method_map = [151    ...     ('**/templates/**.*', 'genshi'),152    ...     ('**.py', 'python')153    ... ]154 155    The dictionary provided by the optional `options_map` parameter augments156    these mappings. It uses extended glob patterns as keys, and the values are157    dictionaries mapping options names to option values (both strings).158 159    The glob patterns of the `options_map` do not necessarily need to be the160    same as those used in the method mapping. For example, while all files in161    the ``templates`` folders in an application may be Genshi applications, the162    options for those files may differ based on extension:163 164    >>> options_map = {165    ...     '**/templates/**.txt': {166    ...         'template_class': 'genshi.template:TextTemplate',167    ...         'encoding': 'latin-1'168    ...     },169    ...     '**/templates/**.html': {170    ...         'include_attrs': ''171    ...     }172    ... }173 174    :param dirname: the path to the directory to extract messages from.  If175                    not given the current working directory is used.176    :param method_map: a list of ``(pattern, method)`` tuples that maps of177                       extraction method names to extended glob patterns178    :param options_map: a dictionary of additional options (optional)179    :param keywords: a dictionary mapping keywords (i.e. names of functions180                     that should be recognized as translation functions) to181                     tuples that specify which of their arguments contain182                     localizable strings183    :param comment_tags: a list of tags of translator comments to search for184                         and include in the results185    :param callback: a function that is called for every file that message are186                     extracted from, just before the extraction itself is187                     performed; the function is passed the filename, the name188                     of the extraction method and and the options dictionary as189                     positional arguments, in that order190    :param strip_comment_tags: a flag that if set to `True` causes all comment191                               tags to be removed from the collected comments.192    :param directory_filter: a callback to determine whether a directory should193                             be recursed into. Receives the full directory path;194                             should return True if the directory is valid.195    :see: `pathmatch`196    """197    if dirname is None:198        dirname = os.getcwd()199    if options_map is None:200        options_map = {}201    if directory_filter is None:202        directory_filter = default_directory_filter203 204    absname = os.path.abspath(dirname)205    for root, dirnames, filenames in os.walk(absname):206        dirnames[:] = [207            subdir for subdir in dirnames208            if directory_filter(os.path.join(root, subdir))209        ]210        dirnames.sort()211        filenames.sort()212        for filename in filenames:213            filepath = os.path.join(root, filename).replace(os.sep, '/')214 215            yield from check_and_call_extract_file(216                filepath,217                method_map,218                options_map,219                callback,220                keywords,221                comment_tags,222                strip_comment_tags,223                dirpath=absname,224            )225 226 227def check_and_call_extract_file(228    filepath: str | os.PathLike[str],229    method_map: Iterable[tuple[str, str]],230    options_map: SupportsItems[str, dict[str, Any]],231    callback: Callable[[str, str, dict[str, Any]], object] | None,232    keywords: Mapping[str, _Keyword],233    comment_tags: Collection[str],234    strip_comment_tags: bool,235    dirpath: str | os.PathLike[str] | None = None,236) -> Generator[_FileExtractionResult, None, None]:237    """Checks if the given file matches an extraction method mapping, and if so, calls extract_from_file.238 239    Note that the extraction method mappings are based relative to dirpath.240    So, given an absolute path to a file `filepath`, we want to check using241    just the relative path from `dirpath` to `filepath`.242 243    Yields 5-tuples (filename, lineno, messages, comments, context).244 245    :param filepath: An absolute path to a file that exists.246    :param method_map: a list of ``(pattern, method)`` tuples that maps of247                       extraction method names to extended glob patterns248    :param options_map: a dictionary of additional options (optional)249    :param callback: a function that is called for every file that message are250                     extracted from, just before the extraction itself is251                     performed; the function is passed the filename, the name252                     of the extraction method and and the options dictionary as253                     positional arguments, in that order254    :param keywords: a dictionary mapping keywords (i.e. names of functions255                     that should be recognized as translation functions) to256                     tuples that specify which of their arguments contain257                     localizable strings258    :param comment_tags: a list of tags of translator comments to search for259                         and include in the results260    :param strip_comment_tags: a flag that if set to `True` causes all comment261                               tags to be removed from the collected comments.262    :param dirpath: the path to the directory to extract messages from.263    :return: iterable of 5-tuples (filename, lineno, messages, comments, context)264    :rtype: Iterable[tuple[str, int, str|tuple[str], list[str], str|None]265    """266    # filename is the relative path from dirpath to the actual file267    filename = relpath(filepath, dirpath)268 269    for pattern, method in method_map:270        if not pathmatch(pattern, filename):271            continue272 273        options = {}274        for opattern, odict in options_map.items():275            if pathmatch(opattern, filename):276                options = odict277        if callback:278            callback(filename, method, options)279        for message_tuple in extract_from_file(280            method, filepath,281            keywords=keywords,282            comment_tags=comment_tags,283            options=options,284            strip_comment_tags=strip_comment_tags,285        ):286            yield (filename, *message_tuple)287 288        break289 290 291def extract_from_file(292    method: _ExtractionMethod,293    filename: str | os.PathLike[str],294    keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,295    comment_tags: Collection[str] = (),296    options: Mapping[str, Any] | None = None,297    strip_comment_tags: bool = False,298) -> list[_ExtractionResult]:299    """Extract messages from a specific file.300 301    This function returns a list of tuples of the form ``(lineno, message, comments, context)``.302 303    :param filename: the path to the file to extract messages from304    :param method: a string specifying the extraction method (.e.g. "python")305    :param keywords: a dictionary mapping keywords (i.e. names of functions306                     that should be recognized as translation functions) to307                     tuples that specify which of their arguments contain308                     localizable strings309    :param comment_tags: a list of translator tags to search for and include310                         in the results311    :param strip_comment_tags: a flag that if set to `True` causes all comment312                               tags to be removed from the collected comments.313    :param options: a dictionary of additional options (optional)314    :returns: list of tuples of the form ``(lineno, message, comments, context)``315    :rtype: list[tuple[int, str|tuple[str], list[str], str|None]316    """317    if method == 'ignore':318        return []319 320    with open(filename, 'rb') as fileobj:321        return list(extract(method, fileobj, keywords, comment_tags,322                            options, strip_comment_tags))323 324 325def _match_messages_against_spec(lineno: int, messages: list[str|None], comments: list[str],326                                 fileobj: _FileObj, spec: tuple[int|tuple[int, str], ...]):327    translatable = []328    context = None329 330    # last_index is 1 based like the keyword spec331    last_index = len(messages)332    for index in spec:333        if isinstance(index, tuple): # (n, 'c')334            context = messages[index[0] - 1]335            continue336        if last_index < index:337            # Not enough arguments338            return339        message = messages[index - 1]340        if message is None:341            return342        translatable.append(message)343 344    # keyword spec indexes are 1 based, therefore '-1'345    if isinstance(spec[0], tuple):346        # context-aware *gettext method347        first_msg_index = spec[1] - 1348    else:349        first_msg_index = spec[0] - 1350    # An empty string msgid isn't valid, emit a warning351    if not messages[first_msg_index]:352        filename = (getattr(fileobj, "name", None) or "(unknown)")353        sys.stderr.write(354            f"{filename}:{lineno}: warning: Empty msgid.  It is reserved by GNU gettext: gettext(\"\") "355            f"returns the header entry with meta information, not the empty string.\n",356        )357        return358 359    translatable = tuple(translatable)360    if len(translatable) == 1:361        translatable = translatable[0]362 363    return lineno, translatable, comments, context364 365 366def extract(367    method: _ExtractionMethod,368    fileobj: _FileObj,369    keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,370    comment_tags: Collection[str] = (),371    options: Mapping[str, Any] | None = None,372    strip_comment_tags: bool = False,373) -> Generator[_ExtractionResult, None, None]:374    """Extract messages from the given file-like object using the specified375    extraction method.376 377    This function returns tuples of the form ``(lineno, message, comments, context)``.378 379    The implementation dispatches the actual extraction to plugins, based on the380    value of the ``method`` parameter.381 382    >>> source = b'''# foo module383    ... def run(argv):384    ...    print(_('Hello, world!'))385    ... '''386 387    >>> from io import BytesIO388    >>> for message in extract('python', BytesIO(source)):389    ...     print(message)390    (3, u'Hello, world!', [], None)391 392    :param method: an extraction method (a callable), or393                   a string specifying the extraction method (.e.g. "python");394                   if this is a simple name, the extraction function will be395                   looked up by entry point; if it is an explicit reference396                   to a function (of the form ``package.module:funcname`` or397                   ``package.module.funcname``), the corresponding function398                   will be imported and used399    :param fileobj: the file-like object the messages should be extracted from400    :param keywords: a dictionary mapping keywords (i.e. names of functions401                     that should be recognized as translation functions) to402                     tuples that specify which of their arguments contain403                     localizable strings404    :param comment_tags: a list of translator tags to search for and include405                         in the results406    :param options: a dictionary of additional options (optional)407    :param strip_comment_tags: a flag that if set to `True` causes all comment408                               tags to be removed from the collected comments.409    :raise ValueError: if the extraction method is not registered410    :returns: iterable of tuples of the form ``(lineno, message, comments, context)``411    :rtype: Iterable[tuple[int, str|tuple[str], list[str], str|None]412    """413    func = None414    if callable(method):415        func = method416    elif ':' in method or '.' in method:417        if ':' not in method:418            lastdot = method.rfind('.')419            module, attrname = method[:lastdot], method[lastdot + 1:]420        else:421            module, attrname = method.split(':', 1)422        func = getattr(__import__(module, {}, {}, [attrname]), attrname)423    else:424        try:425            from pkg_resources import working_set426        except ImportError:427            pass428        else:429            for entry_point in working_set.iter_entry_points(GROUP_NAME,430                                                             method):431                func = entry_point.load(require=True)432                break433        if func is None:434            # if pkg_resources is not available or no usable egg-info was found435            # (see #230), we resort to looking up the builtin extractors436            # directly437            builtin = {438                'ignore': extract_nothing,439                'python': extract_python,440                'javascript': extract_javascript,441            }442            func = builtin.get(method)443 444    if func is None:445        raise ValueError(f"Unknown extraction method {method!r}")446 447    results = func(fileobj, keywords.keys(), comment_tags,448                   options=options or {})449 450    for lineno, funcname, messages, comments in results:451        if not isinstance(messages, (list, tuple)):452            messages = [messages]453        if not messages:454            continue455 456        specs = keywords[funcname] or None if funcname else None457        # {None: x} may be collapsed into x for backwards compatibility.458        if not isinstance(specs, dict):459            specs = {None: specs}460 461        if strip_comment_tags:462            _strip_comment_tags(comments, comment_tags)463 464        # None matches all arities.465        for arity in (None, len(messages)):466            try:467                spec = specs[arity]468            except KeyError:469                continue470            if spec is None:471                spec = (1,)472            result = _match_messages_against_spec(lineno, messages, comments, fileobj, spec)473            if result is not None:474                yield result475 476 477def extract_nothing(478    fileobj: _FileObj,479    keywords: Mapping[str, _Keyword],480    comment_tags: Collection[str],481    options: Mapping[str, Any],482) -> list[_ExtractionResult]:483    """Pseudo extractor that does not actually extract anything, but simply484    returns an empty list.485    """486    return []487 488 489def extract_python(490    fileobj: IO[bytes],491    keywords: Mapping[str, _Keyword],492    comment_tags: Collection[str],493    options: _PyOptions,494) -> Generator[_ExtractionResult, None, None]:495    """Extract messages from Python source code.496 497    It returns an iterator yielding tuples in the following form ``(lineno,498    funcname, message, comments)``.499 500    :param fileobj: the seekable, file-like object the messages should be501                    extracted from502    :param keywords: a list of keywords (i.e. function names) that should be503                     recognized as translation functions504    :param comment_tags: a list of translator tags to search for and include505                         in the results506    :param options: a dictionary of additional options (optional)507    :rtype: ``iterator``508    """509    funcname = lineno = message_lineno = None510    call_stack = -1511    buf = []512    messages = []513    translator_comments = []514    in_def = in_translator_comments = False515    comment_tag = None516 517    encoding = parse_encoding(fileobj) or options.get('encoding', 'UTF-8')518    future_flags = parse_future_flags(fileobj, encoding)519    next_line = lambda: fileobj.readline().decode(encoding)520 521    tokens = generate_tokens(next_line)522 523    # Current prefix of a Python 3.12 (PEP 701) f-string, or None if we're not524    # currently parsing one.525    current_fstring_start = None526 527    for tok, value, (lineno, _), _, _ in tokens:528        if call_stack == -1 and tok == NAME and value in ('def', 'class'):529            in_def = True530        elif tok == OP and value == '(':531            if in_def:532                # Avoid false positives for declarations such as:533                # def gettext(arg='message'):534                in_def = False535                continue536            if funcname:537                message_lineno = lineno538                call_stack += 1539        elif in_def and tok == OP and value == ':':540            # End of a class definition without parens541            in_def = False542            continue543        elif call_stack == -1 and tok == COMMENT:544            # Strip the comment token from the line545            value = value[1:].strip()546            if in_translator_comments and \547                    translator_comments[-1][0] == lineno - 1:548                # We're already inside a translator comment, continue appending549                translator_comments.append((lineno, value))550                continue551            # If execution reaches this point, let's see if comment line552            # starts with one of the comment tags553            for comment_tag in comment_tags:554                if value.startswith(comment_tag):555                    in_translator_comments = True556                    translator_comments.append((lineno, value))557                    break558        elif funcname and call_stack == 0:559            nested = (tok == NAME and value in keywords)560            if (tok == OP and value == ')') or nested:561                if buf:562                    messages.append(''.join(buf))563                    del buf[:]564                else:565                    messages.append(None)566 567                messages = tuple(messages) if len(messages) > 1 else messages[0]568                # Comments don't apply unless they immediately569                # precede the message570                if translator_comments and \571                        translator_comments[-1][0] < message_lineno - 1:572                    translator_comments = []573 574                yield (message_lineno, funcname, messages,575                       [comment[1] for comment in translator_comments])576 577                funcname = lineno = message_lineno = None578                call_stack = -1579                messages = []580                translator_comments = []581                in_translator_comments = False582                if nested:583                    funcname = value584            elif tok == STRING:585                val = _parse_python_string(value, encoding, future_flags)586                if val is not None:587                    buf.append(val)588 589            # Python 3.12+, see https://peps.python.org/pep-0701/#new-tokens590            elif tok == FSTRING_START:591                current_fstring_start = value592            elif tok == FSTRING_MIDDLE:593                if current_fstring_start is not None:594                    current_fstring_start += value595            elif tok == FSTRING_END:596                if current_fstring_start is not None:597                    fstring = current_fstring_start + value598                    val = _parse_python_string(fstring, encoding, future_flags)599                    if val is not None:600                        buf.append(val)601 602            elif tok == OP and value == ',':603                if buf:604                    messages.append(''.join(buf))605                    del buf[:]606                else:607                    messages.append(None)608                if translator_comments:609                    # We have translator comments, and since we're on a610                    # comma(,) user is allowed to break into a new line611                    # Let's increase the last comment's lineno in order612                    # for the comment to still be a valid one613                    old_lineno, old_comment = translator_comments.pop()614                    translator_comments.append((old_lineno + 1, old_comment))615        elif call_stack > 0 and tok == OP and value == ')':616            call_stack -= 1617        elif funcname and call_stack == -1:618            funcname = None619        elif tok == NAME and value in keywords:620            funcname = value621 622        if (current_fstring_start is not None623            and tok not in {FSTRING_START, FSTRING_MIDDLE}624        ):625            # In Python 3.12, tokens other than FSTRING_* mean the626            # f-string is dynamic, so we don't wan't to extract it.627            # And if it's FSTRING_END, we've already handled it above.628            # Let's forget that we're in an f-string.629            current_fstring_start = None630 631 632def _parse_python_string(value: str, encoding: str, future_flags: int) -> str | None:633    # Unwrap quotes in a safe manner, maintaining the string's encoding634    # https://sourceforge.net/tracker/?func=detail&atid=355470&aid=617979&group_id=5470635    code = compile(636        f'# coding={str(encoding)}\n{value}',637        '<string>',638        'eval',639        ast.PyCF_ONLY_AST | future_flags,640    )641    if isinstance(code, ast.Expression):642        body = code.body643        if isinstance(body, ast.Str):644            return body.s645        if isinstance(body, ast.JoinedStr):  # f-string646            if all(isinstance(node, ast.Str) for node in body.values):647                return ''.join(node.s for node in body.values)648            if all(isinstance(node, ast.Constant) for node in body.values):649                return ''.join(str(node.value) for node in body.values)650            # TODO: we could raise an error or warning when not all nodes are constants651    return None652 653 654def extract_javascript(655    fileobj: _FileObj,656    keywords: Mapping[str, _Keyword],657    comment_tags: Collection[str],658    options: _JSOptions,659    lineno: int = 1,660) -> Generator[_ExtractionResult, None, None]:661    """Extract messages from JavaScript source code.662 663    :param fileobj: the seekable, file-like object the messages should be664                    extracted from665    :param keywords: a list of keywords (i.e. function names) that should be666                     recognized as translation functions667    :param comment_tags: a list of translator tags to search for and include668                         in the results669    :param options: a dictionary of additional options (optional)670                    Supported options are:671                    * `jsx` -- set to false to disable JSX/E4X support.672                    * `template_string` -- if `True`, supports gettext(`key`)673                    * `parse_template_string` -- if `True` will parse the674                                                 contents of javascript675                                                 template strings.676    :param lineno: line number offset (for parsing embedded fragments)677    """678    from babel.messages.jslexer import Token, tokenize, unquote_string679    funcname = message_lineno = None680    messages = []681    last_argument = None682    translator_comments = []683    concatenate_next = False684    encoding = options.get('encoding', 'utf-8')685    last_token = None686    call_stack = -1687    dotted = any('.' in kw for kw in keywords)688    for token in tokenize(689        fileobj.read().decode(encoding),690        jsx=options.get("jsx", True),691        template_string=options.get("template_string", True),692        dotted=dotted,693        lineno=lineno,694    ):695        if (  # Turn keyword`foo` expressions into keyword("foo") calls:696            funcname and  # have a keyword...697            (last_token and last_token.type == 'name') and  # we've seen nothing after the keyword...698            token.type == 'template_string'  # this is a template string699        ):700            message_lineno = token.lineno701            messages = [unquote_string(token.value)]702            call_stack = 0703            token = Token('operator', ')', token.lineno)704 705        if options.get('parse_template_string') and not funcname and token.type == 'template_string':706            yield from parse_template_string(token.value, keywords, comment_tags, options, token.lineno)707 708        elif token.type == 'operator' and token.value == '(':709            if funcname:710                message_lineno = token.lineno711                call_stack += 1712 713        elif call_stack == -1 and token.type == 'linecomment':714            value = token.value[2:].strip()715            if translator_comments and \716               translator_comments[-1][0] == token.lineno - 1:717                translator_comments.append((token.lineno, value))718                continue719 720            for comment_tag in comment_tags:721                if value.startswith(comment_tag):722                    translator_comments.append((token.lineno, value.strip()))723                    break724 725        elif token.type == 'multilinecomment':726            # only one multi-line comment may precede a translation727            translator_comments = []728            value = token.value[2:-2].strip()729            for comment_tag in comment_tags:730                if value.startswith(comment_tag):731                    lines = value.splitlines()732                    if lines:733                        lines[0] = lines[0].strip()734                        lines[1:] = dedent('\n'.join(lines[1:])).splitlines()735                        for offset, line in enumerate(lines):736                            translator_comments.append((token.lineno + offset,737                                                        line))738                    break739 740        elif funcname and call_stack == 0:741            if token.type == 'operator' and token.value == ')':742                if last_argument is not None:743                    messages.append(last_argument)744                if len(messages) > 1:745                    messages = tuple(messages)746                elif messages:747                    messages = messages[0]748                else:749                    messages = None750 751                # Comments don't apply unless they immediately precede the752                # message753                if translator_comments and \754                   translator_comments[-1][0] < message_lineno - 1:755                    translator_comments = []756 757                if messages is not None:758                    yield (message_lineno, funcname, messages,759                           [comment[1] for comment in translator_comments])760 761                funcname = message_lineno = last_argument = None762                concatenate_next = False763                translator_comments = []764                messages = []765                call_stack = -1766 767            elif token.type in ('string', 'template_string'):768                new_value = unquote_string(token.value)769                if concatenate_next:770                    last_argument = (last_argument or '') + new_value771                    concatenate_next = False772                else:773                    last_argument = new_value774 775            elif token.type == 'operator':776                if token.value == ',':777                    if last_argument is not None:778                        messages.append(last_argument)779                        last_argument = None780                    else:781                        messages.append(None)782                    concatenate_next = False783                elif token.value == '+':784                    concatenate_next = True785 786        elif call_stack > 0 and token.type == 'operator' \787                and token.value == ')':788            call_stack -= 1789 790        elif funcname and call_stack == -1:791            funcname = None792 793        elif call_stack == -1 and token.type == 'name' and \794            token.value in keywords and \795            (last_token is None or last_token.type != 'name' or796             last_token.value != 'function'):797            funcname = token.value798 799        last_token = token800 801 802def parse_template_string(803    template_string: str,804    keywords: Mapping[str, _Keyword],805    comment_tags: Collection[str],806    options: _JSOptions,807    lineno: int = 1,808) -> Generator[_ExtractionResult, None, None]:809    """Parse JavaScript template string.810 811    :param template_string: the template string to be parsed812    :param keywords: a list of keywords (i.e. function names) that should be813                     recognized as translation functions814    :param comment_tags: a list of translator tags to search for and include815                         in the results816    :param options: a dictionary of additional options (optional)817    :param lineno: starting line number (optional)818    """819    from babel.messages.jslexer import line_re820    prev_character = None821    level = 0822    inside_str = False823    expression_contents = ''824    for character in template_string[1:-1]:825        if not inside_str and character in ('"', "'", '`'):826            inside_str = character827        elif inside_str == character and prev_character != r'\\':828            inside_str = False829        if level:830            expression_contents += character831        if not inside_str:832            if character == '{' and prev_character == '$':833                level += 1834            elif level and character == '}':835                level -= 1836                if level == 0 and expression_contents:837                    expression_contents = expression_contents[0:-1]838                    fake_file_obj = io.BytesIO(expression_contents.encode())839                    yield from extract_javascript(fake_file_obj, keywords, comment_tags, options, lineno)840                    lineno += len(line_re.findall(expression_contents))841                    expression_contents = ''842        prev_character = character843