codekingpro/portable-devtools

1# mako/lexer.py2# Copyright 2006-2024 the Mako authors and contributors <see AUTHORS file>3#4# This module is part of Mako and is released under5# the MIT License: http://www.opensource.org/licenses/mit-license.php6 7"""provides the Lexer class for parsing template strings into parse trees."""8 9import codecs10import re11 12from mako import exceptions13from mako import parsetree14from mako.pygen import adjust_whitespace15 16_regexp_cache = {}17 18 19class Lexer:20    def __init__(21        self, text, filename=None, input_encoding=None, preprocessor=None22    ):23        self.text = text24        self.filename = filename25        self.template = parsetree.TemplateNode(self.filename)26        self.matched_lineno = 127        self.matched_charpos = 028        self.lineno = 129        self.match_position = 030        self.tag = []31        self.control_line = []32        self.ternary_stack = []33        self.encoding = input_encoding34 35        if preprocessor is None:36            self.preprocessor = []37        elif not hasattr(preprocessor, "__iter__"):38            self.preprocessor = [preprocessor]39        else:40            self.preprocessor = preprocessor41 42    @property43    def exception_kwargs(self):44        return {45            "source": self.text,46            "lineno": self.matched_lineno,47            "pos": self.matched_charpos,48            "filename": self.filename,49        }50 51    def match(self, regexp, flags=None):52        """compile the given regexp, cache the reg, and call match_reg()."""53 54        try:55            reg = _regexp_cache[(regexp, flags)]56        except KeyError:57            reg = re.compile(regexp, flags) if flags else re.compile(regexp)58            _regexp_cache[(regexp, flags)] = reg59 60        return self.match_reg(reg)61 62    def match_reg(self, reg):63        """match the given regular expression object to the current text64        position.65 66        if a match occurs, update the current text and line position.67 68        """69 70        mp = self.match_position71 72        match = reg.match(self.text, self.match_position)73        if match:74            (start, end) = match.span()75            self.match_position = end + 1 if end == start else end76            self.matched_lineno = self.lineno77            cp = mp - 178            if cp >= 0 and cp < self.textlength:79                cp = self.text[: cp + 1].rfind("\n")80            self.matched_charpos = mp - cp81            self.lineno += self.text[mp : self.match_position].count("\n")82        return match83 84    def parse_until_text(self, watch_nesting, *text):85        startpos = self.match_position86        text_re = r"|".join(text)87        brace_level = 088        paren_level = 089        bracket_level = 090        while True:91            match = self.match(r"#.*\n")92            if match:93                continue94            match = self.match(95                r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S96            )97            if match:98                continue99            match = self.match(r"(%s)" % text_re)100            if match and not (101                watch_nesting102                and (brace_level > 0 or paren_level > 0 or bracket_level > 0)103            ):104                return (105                    self.text[106                        startpos : self.match_position - len(match.group(1))107                    ],108                    match.group(1),109                )110            elif not match:111                match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S)112            if match:113                brace_level += match.group(1).count("{")114                brace_level -= match.group(1).count("}")115                paren_level += match.group(1).count("(")116                paren_level -= match.group(1).count(")")117                bracket_level += match.group(1).count("[")118                bracket_level -= match.group(1).count("]")119                continue120            raise exceptions.SyntaxException(121                "Expected: %s" % ",".join(text), **self.exception_kwargs122            )123 124    def append_node(self, nodecls, *args, **kwargs):125        kwargs.setdefault("source", self.text)126        kwargs.setdefault("lineno", self.matched_lineno)127        kwargs.setdefault("pos", self.matched_charpos)128        kwargs["filename"] = self.filename129        node = nodecls(*args, **kwargs)130        if len(self.tag):131            self.tag[-1].nodes.append(node)132        else:133            self.template.nodes.append(node)134        # build a set of child nodes for the control line135        # (used for loop variable detection)136        # also build a set of child nodes on ternary control lines137        # (used for determining if a pass needs to be auto-inserted138        if self.control_line:139            control_frame = self.control_line[-1]140            control_frame.nodes.append(node)141            if (142                not (143                    isinstance(node, parsetree.ControlLine)144                    and control_frame.is_ternary(node.keyword)145                )146                and self.ternary_stack147                and self.ternary_stack[-1]148            ):149                self.ternary_stack[-1][-1].nodes.append(node)150        if isinstance(node, parsetree.Tag):151            if len(self.tag):152                node.parent = self.tag[-1]153            self.tag.append(node)154        elif isinstance(node, parsetree.ControlLine):155            if node.isend:156                self.control_line.pop()157                self.ternary_stack.pop()158            elif node.is_primary:159                self.control_line.append(node)160                self.ternary_stack.append([])161            elif self.control_line and self.control_line[-1].is_ternary(162                node.keyword163            ):164                self.ternary_stack[-1].append(node)165            elif self.control_line and not self.control_line[-1].is_ternary(166                node.keyword167            ):168                raise exceptions.SyntaxException(169                    "Keyword '%s' not a legal ternary for keyword '%s'"170                    % (node.keyword, self.control_line[-1].keyword),171                    **self.exception_kwargs,172                )173 174    _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n")175 176    def decode_raw_stream(self, text, decode_raw, known_encoding, filename):177        """given string/unicode or bytes/string, determine encoding178        from magic encoding comment, return body as unicode179        or raw if decode_raw=False180 181        """182        if isinstance(text, str):183            m = self._coding_re.match(text)184            encoding = m and m.group(1) or known_encoding or "utf-8"185            return encoding, text186 187        if text.startswith(codecs.BOM_UTF8):188            text = text[len(codecs.BOM_UTF8) :]189            parsed_encoding = "utf-8"190            m = self._coding_re.match(text.decode("utf-8", "ignore"))191            if m is not None and m.group(1) != "utf-8":192                raise exceptions.CompileException(193                    "Found utf-8 BOM in file, with conflicting "194                    "magic encoding comment of '%s'" % m.group(1),195                    text.decode("utf-8", "ignore"),196                    0,197                    0,198                    filename,199                )200        else:201            m = self._coding_re.match(text.decode("utf-8", "ignore"))202            parsed_encoding = m.group(1) if m else known_encoding or "utf-8"203        if decode_raw:204            try:205                text = text.decode(parsed_encoding)206            except UnicodeDecodeError:207                raise exceptions.CompileException(208                    "Unicode decode operation of encoding '%s' failed"209                    % parsed_encoding,210                    text.decode("utf-8", "ignore"),211                    0,212                    0,213                    filename,214                )215 216        return parsed_encoding, text217 218    def parse(self):219        self.encoding, self.text = self.decode_raw_stream(220            self.text, True, self.encoding, self.filename221        )222 223        for preproc in self.preprocessor:224            self.text = preproc(self.text)225 226        # push the match marker past the227        # encoding comment.228        self.match_reg(self._coding_re)229 230        self.textlength = len(self.text)231 232        while True:233            if self.match_position > self.textlength:234                break235 236            if self.match_end():237                break238            if self.match_expression():239                continue240            if self.match_control_line():241                continue242            if self.match_comment():243                continue244            if self.match_tag_start():245                continue246            if self.match_tag_end():247                continue248            if self.match_python_block():249                continue250            if self.match_percent():251                continue252            if self.match_text():253                continue254 255            if self.match_position > self.textlength:256                break257            # TODO: no coverage here258            raise exceptions.MakoException("assertion failed")259 260        if len(self.tag):261            raise exceptions.SyntaxException(262                "Unclosed tag: <%%%s>" % self.tag[-1].keyword,263                **self.exception_kwargs,264            )265        if len(self.control_line):266            raise exceptions.SyntaxException(267                "Unterminated control keyword: '%s'"268                % self.control_line[-1].keyword,269                self.text,270                self.control_line[-1].lineno,271                self.control_line[-1].pos,272                self.filename,273            )274        return self.template275 276    def match_tag_start(self):277        reg = r"""278            \<%     # opening tag279 280            ([\w\.\:]+)   # keyword281 282            ((?:\s+\w+|\s*=\s*|"[^"]*?"|'[^']*?'|\s*,\s*)*)  # attrname, = \283                                               #        sign, string expression284                                               # comma is for backwards compat285                                               # identified in #366286 287            \s*     # more whitespace288 289            (/)?>   # closing290 291        """292 293        match = self.match(294            reg,295            re.I | re.S | re.X,296        )297 298        if not match:299            return False300 301        keyword, attr, isend = match.groups()302        self.keyword = keyword303        attributes = {}304        if attr:305            for att in re.findall(306                r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr307            ):308                key, val1, val2 = att309                text = val1 or val2310                text = text.replace("\r\n", "\n")311                attributes[key] = text312        self.append_node(parsetree.Tag, keyword, attributes)313        if isend:314            self.tag.pop()315        elif keyword == "text":316            match = self.match(r"(.*?)(?=\</%text>)", re.S)317            if not match:318                raise exceptions.SyntaxException(319                    "Unclosed tag: <%%%s>" % self.tag[-1].keyword,320                    **self.exception_kwargs,321                )322            self.append_node(parsetree.Text, match.group(1))323            return self.match_tag_end()324        return True325 326    def match_tag_end(self):327        match = self.match(r"\</%[\t ]*([^\t ]+?)[\t ]*>")328        if match:329            if not len(self.tag):330                raise exceptions.SyntaxException(331                    "Closing tag without opening tag: </%%%s>"332                    % match.group(1),333                    **self.exception_kwargs,334                )335            elif self.tag[-1].keyword != match.group(1):336                raise exceptions.SyntaxException(337                    "Closing tag </%%%s> does not match tag: <%%%s>"338                    % (match.group(1), self.tag[-1].keyword),339                    **self.exception_kwargs,340                )341            self.tag.pop()342            return True343        else:344            return False345 346    def match_end(self):347        match = self.match(r"\Z", re.S)348        if not match:349            return False350 351        string = match.group()352        if string:353            return string354        else:355            return True356 357    def match_percent(self):358        match = self.match(r"(?<=^)(\s*)%%(%*)", re.M)359        if match:360            self.append_node(361                parsetree.Text, match.group(1) + "%" + match.group(2)362            )363            return True364        else:365            return False366 367    def match_text(self):368        match = self.match(369            r"""370                (.*?)         # anything, followed by:371                (372                 (?<=\n)(?=[ \t]*(?=%|\#\#))  # an eval or line-based373                                            # comment, preceded by a374                                            # consumed newline and whitespace375                 |376                 (?=\${)      # an expression377                 |378                 (?=</?[%&])  # a substitution or block or call start or end379                              # - don't consume380                 |381                 (\\\r?\n)    # an escaped newline  - throw away382                 |383                 \Z           # end of string384                )""",385            re.X | re.S,386        )387 388        if match:389            text = match.group(1)390            if text:391                self.append_node(parsetree.Text, text)392            return True393        else:394            return False395 396    def match_python_block(self):397        match = self.match(r"<%(!)?")398        if match:399            line, pos = self.matched_lineno, self.matched_charpos400            text, end = self.parse_until_text(False, r"%>")401            # the trailing newline helps402            # compiler.parse() not complain about indentation403            text = adjust_whitespace(text) + "\n"404            self.append_node(405                parsetree.Code,406                text,407                match.group(1) == "!",408                lineno=line,409                pos=pos,410            )411            return True412        else:413            return False414 415    def match_expression(self):416        match = self.match(r"\${")417        if not match:418            return False419 420        line, pos = self.matched_lineno, self.matched_charpos421        text, end = self.parse_until_text(True, r"\|", r"}")422        if end == "|":423            escapes, end = self.parse_until_text(True, r"}")424        else:425            escapes = ""426        text = text.replace("\r\n", "\n")427        self.append_node(428            parsetree.Expression,429            text,430            escapes.strip(),431            lineno=line,432            pos=pos,433        )434        return True435 436    def match_control_line(self):437        match = self.match(438            r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\\r?\n)|[^\r\n])*)"439            r"(?:\r?\n|\Z)",440            re.M,441        )442        if not match:443            return False444 445        operator = match.group(1)446        text = match.group(2)447        if operator == "%":448            m2 = re.match(r"(end)?(\w+)\s*(.*)", text)449            if not m2:450                raise exceptions.SyntaxException(451                    "Invalid control line: '%s'" % text,452                    **self.exception_kwargs,453                )454            isend, keyword = m2.group(1, 2)455            isend = isend is not None456 457            if isend:458                if not len(self.control_line):459                    raise exceptions.SyntaxException(460                        "No starting keyword '%s' for '%s'" % (keyword, text),461                        **self.exception_kwargs,462                    )463                elif self.control_line[-1].keyword != keyword:464                    raise exceptions.SyntaxException(465                        "Keyword '%s' doesn't match keyword '%s'"466                        % (text, self.control_line[-1].keyword),467                        **self.exception_kwargs,468                    )469            self.append_node(parsetree.ControlLine, keyword, isend, text)470        else:471            self.append_node(parsetree.Comment, text)472        return True473 474    def match_comment(self):475        """matches the multiline version of a comment"""476        match = self.match(r"<%doc>(.*?)</%doc>", re.S)477        if match:478            self.append_node(parsetree.Comment, match.group(1))479            return True480        else:481            return False482