codekingpro/portable-devtools

1"""Define partial Python code Parser used by editor and hyperparser.2 3Instances of ParseMap are used with str.translate.4 5The following bound search and match functions are defined:6_synchre - start of popular statement;7_junkre - whitespace or comment line;8_match_stringre: string, possibly without closer;9_itemre - line that may have bracket structure start;10_closere - line that must be followed by dedent.11_chew_ordinaryre - non-special characters.12"""13import re14 15# Reason last statement is continued (or C_NONE if it's not).16(C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,17 C_STRING_NEXT_LINES, C_BRACKET) = range(5)18 19# Find what looks like the start of a popular statement.20 21_synchre = re.compile(r"""22    ^23    [ \t]*24    (?: while25    |   else26    |   def27    |   return28    |   assert29    |   break30    |   class31    |   continue32    |   elif33    |   try34    |   except35    |   raise36    |   import37    |   yield38    )39    \b40""", re.VERBOSE | re.MULTILINE).search41 42# Match blank line or non-indenting comment line.43 44_junkre = re.compile(r"""45    [ \t]*46    (?: \# \S .* )?47    \n48""", re.VERBOSE).match49 50# Match any flavor of string; the terminating quote is optional51# so that we're robust in the face of incomplete program text.52 53_match_stringre = re.compile(r"""54    \""" [^"\\]* (?:55                     (?: \\. | "(?!"") )56                     [^"\\]*57                 )*58    (?: \""" )?59 60|   " [^"\\\n]* (?: \\. [^"\\\n]* )* "?61 62|   ''' [^'\\]* (?:63                   (?: \\. | '(?!'') )64                   [^'\\]*65                )*66    (?: ''' )?67 68|   ' [^'\\\n]* (?: \\. [^'\\\n]* )* '?69""", re.VERBOSE | re.DOTALL).match70 71# Match a line that starts with something interesting;72# used to find the first item of a bracket structure.73 74_itemre = re.compile(r"""75    [ \t]*76    [^\s#\\]    # if we match, m.end()-1 is the interesting char77""", re.VERBOSE).match78 79# Match start of statements that should be followed by a dedent.80 81_closere = re.compile(r"""82    \s*83    (?: return84    |   break85    |   continue86    |   raise87    |   pass88    )89    \b90""", re.VERBOSE).match91 92# Chew up non-special chars as quickly as possible.  If match is93# successful, m.end() less 1 is the index of the last boring char94# matched.  If match is unsuccessful, the string starts with an95# interesting char.96 97_chew_ordinaryre = re.compile(r"""98    [^[\](){}#'"\\]+99""", re.VERBOSE).match100 101 102class ParseMap(dict):103    r"""Dict subclass that maps anything not in dict to 'x'.104 105    This is designed to be used with str.translate in study1.106    Anything not specifically mapped otherwise becomes 'x'.107    Example: replace everything except whitespace with 'x'.108 109    >>> keepwhite = ParseMap((ord(c), ord(c)) for c in ' \t\n\r')110    >>> "a + b\tc\nd".translate(keepwhite)111    'x x x\tx\nx'112    """113    # Calling this triples access time; see bpo-32940114    def __missing__(self, key):115        return 120  # ord('x')116 117 118# Map all ascii to 120 to avoid __missing__ call, then replace some.119trans = ParseMap.fromkeys(range(128), 120)120trans.update((ord(c), ord('(')) for c in "({[")  # open brackets => '(';121trans.update((ord(c), ord(')')) for c in ")}]")  # close brackets => ')'.122trans.update((ord(c), ord(c)) for c in "\"'\\\n#")  # Keep these.123 124 125class Parser:126 127    def __init__(self, indentwidth, tabwidth):128        self.indentwidth = indentwidth129        self.tabwidth = tabwidth130 131    def set_code(self, s):132        assert len(s) == 0 or s[-1] == '\n'133        self.code = s134        self.study_level = 0135 136    def find_good_parse_start(self, is_char_in_string):137        """138        Return index of a good place to begin parsing, as close to the139        end of the string as possible.  This will be the start of some140        popular stmt like "if" or "def".  Return None if none found:141        the caller should pass more prior context then, if possible, or142        if not (the entire program text up until the point of interest143        has already been tried) pass 0 to set_lo().144 145        This will be reliable iff given a reliable is_char_in_string()146        function, meaning that when it says "no", it's absolutely147        guaranteed that the char is not in a string.148        """149        code, pos = self.code, None150 151        # Peek back from the end for a good place to start,152        # but don't try too often; pos will be left None, or153        # bumped to a legitimate synch point.154        limit = len(code)155        for tries in range(5):156            i = code.rfind(":\n", 0, limit)157            if i < 0:158                break159            i = code.rfind('\n', 0, i) + 1  # start of colon line (-1+1=0)160            m = _synchre(code, i, limit)161            if m and not is_char_in_string(m.start()):162                pos = m.start()163                break164            limit = i165        if pos is None:166            # Nothing looks like a block-opener, or stuff does167            # but is_char_in_string keeps returning true; most likely168            # we're in or near a giant string, the colorizer hasn't169            # caught up enough to be helpful, or there simply *aren't*170            # any interesting stmts.  In any of these cases we're171            # going to have to parse the whole thing to be sure, so172            # give it one last try from the start, but stop wasting173            # time here regardless of the outcome.174            m = _synchre(code)175            if m and not is_char_in_string(m.start()):176                pos = m.start()177            return pos178 179        # Peeking back worked; look forward until _synchre no longer180        # matches.181        i = pos + 1182        while m := _synchre(code, i):183            s, i = m.span()184            if not is_char_in_string(s):185                pos = s186        return pos187 188    def set_lo(self, lo):189        """ Throw away the start of the string.190 191        Intended to be called with the result of find_good_parse_start().192        """193        assert lo == 0 or self.code[lo-1] == '\n'194        if lo > 0:195            self.code = self.code[lo:]196 197    def _study1(self):198        """Find the line numbers of non-continuation lines.199 200        As quickly as humanly possible <wink>, find the line numbers (0-201        based) of the non-continuation lines.202        Creates self.{goodlines, continuation}.203        """204        if self.study_level >= 1:205            return206        self.study_level = 1207 208        # Map all uninteresting characters to "x", all open brackets209        # to "(", all close brackets to ")", then collapse runs of210        # uninteresting characters.  This can cut the number of chars211        # by a factor of 10-40, and so greatly speed the following loop.212        code = self.code213        code = code.translate(trans)214        code = code.replace('xxxxxxxx', 'x')215        code = code.replace('xxxx', 'x')216        code = code.replace('xx', 'x')217        code = code.replace('xx', 'x')218        code = code.replace('\nx', '\n')219        # Replacing x\n with \n would be incorrect because220        # x may be preceded by a backslash.221 222        # March over the squashed version of the program, accumulating223        # the line numbers of non-continued stmts, and determining224        # whether & why the last stmt is a continuation.225        continuation = C_NONE226        level = lno = 0     # level is nesting level; lno is line number227        self.goodlines = goodlines = [0]228        push_good = goodlines.append229        i, n = 0, len(code)230        while i < n:231            ch = code[i]232            i = i+1233 234            # cases are checked in decreasing order of frequency235            if ch == 'x':236                continue237 238            if ch == '\n':239                lno = lno + 1240                if level == 0:241                    push_good(lno)242                    # else we're in an unclosed bracket structure243                continue244 245            if ch == '(':246                level = level + 1247                continue248 249            if ch == ')':250                if level:251                    level = level - 1252                    # else the program is invalid, but we can't complain253                continue254 255            if ch == '"' or ch == "'":256                # consume the string257                quote = ch258                if code[i-1:i+2] == quote * 3:259                    quote = quote * 3260                firstlno = lno261                w = len(quote) - 1262                i = i+w263                while i < n:264                    ch = code[i]265                    i = i+1266 267                    if ch == 'x':268                        continue269 270                    if code[i-1:i+w] == quote:271                        i = i+w272                        break273 274                    if ch == '\n':275                        lno = lno + 1276                        if w == 0:277                            # unterminated single-quoted string278                            if level == 0:279                                push_good(lno)280                            break281                        continue282 283                    if ch == '\\':284                        assert i < n285                        if code[i] == '\n':286                            lno = lno + 1287                        i = i+1288                        continue289 290                    # else comment char or paren inside string291 292                else:293                    # didn't break out of the loop, so we're still294                    # inside a string295                    if (lno - 1) == firstlno:296                        # before the previous \n in code, we were in the first297                        # line of the string298                        continuation = C_STRING_FIRST_LINE299                    else:300                        continuation = C_STRING_NEXT_LINES301                continue    # with outer loop302 303            if ch == '#':304                # consume the comment305                i = code.find('\n', i)306                assert i >= 0307                continue308 309            assert ch == '\\'310            assert i < n311            if code[i] == '\n':312                lno = lno + 1313                if i+1 == n:314                    continuation = C_BACKSLASH315            i = i+1316 317        # The last stmt may be continued for all 3 reasons.318        # String continuation takes precedence over bracket319        # continuation, which beats backslash continuation.320        if (continuation != C_STRING_FIRST_LINE321            and continuation != C_STRING_NEXT_LINES and level > 0):322            continuation = C_BRACKET323        self.continuation = continuation324 325        # Push the final line number as a sentinel value, regardless of326        # whether it's continued.327        assert (continuation == C_NONE) == (goodlines[-1] == lno)328        if goodlines[-1] != lno:329            push_good(lno)330 331    def get_continuation_type(self):332        self._study1()333        return self.continuation334 335    def _study2(self):336        """337        study1 was sufficient to determine the continuation status,338        but doing more requires looking at every character.  study2339        does this for the last interesting statement in the block.340        Creates:341            self.stmt_start, stmt_end342                slice indices of last interesting stmt343            self.stmt_bracketing344                the bracketing structure of the last interesting stmt; for345                example, for the statement "say(boo) or die",346                stmt_bracketing will be ((0, 0), (0, 1), (2, 0), (2, 1),347                (4, 0)). Strings and comments are treated as brackets, for348                the matter.349            self.lastch350                last interesting character before optional trailing comment351            self.lastopenbracketpos352                if continuation is C_BRACKET, index of last open bracket353        """354        if self.study_level >= 2:355            return356        self._study1()357        self.study_level = 2358 359        # Set p and q to slice indices of last interesting stmt.360        code, goodlines = self.code, self.goodlines361        i = len(goodlines) - 1  # Index of newest line.362        p = len(code)  # End of goodlines[i]363        while i:364            assert p365            # Make p be the index of the stmt at line number goodlines[i].366            # Move p back to the stmt at line number goodlines[i-1].367            q = p368            for nothing in range(goodlines[i-1], goodlines[i]):369                # tricky: sets p to 0 if no preceding newline370                p = code.rfind('\n', 0, p-1) + 1371            # The stmt code[p:q] isn't a continuation, but may be blank372            # or a non-indenting comment line.373            if  _junkre(code, p):374                i = i-1375            else:376                break377        if i == 0:378            # nothing but junk!379            assert p == 0380            q = p381        self.stmt_start, self.stmt_end = p, q382 383        # Analyze this stmt, to find the last open bracket (if any)384        # and last interesting character (if any).385        lastch = ""386        stack = []  # stack of open bracket indices387        push_stack = stack.append388        bracketing = [(p, 0)]389        while p < q:390            # suck up all except ()[]{}'"#\\391            m = _chew_ordinaryre(code, p, q)392            if m:393                # we skipped at least one boring char394                newp = m.end()395                # back up over totally boring whitespace396                i = newp - 1    # index of last boring char397                while i >= p and code[i] in " \t\n":398                    i = i-1399                if i >= p:400                    lastch = code[i]401                p = newp402                if p >= q:403                    break404 405            ch = code[p]406 407            if ch in "([{":408                push_stack(p)409                bracketing.append((p, len(stack)))410                lastch = ch411                p = p+1412                continue413 414            if ch in ")]}":415                if stack:416                    del stack[-1]417                lastch = ch418                p = p+1419                bracketing.append((p, len(stack)))420                continue421 422            if ch == '"' or ch == "'":423                # consume string424                # Note that study1 did this with a Python loop, but425                # we use a regexp here; the reason is speed in both426                # cases; the string may be huge, but study1 pre-squashed427                # strings to a couple of characters per line.  study1428                # also needed to keep track of newlines, and we don't429                # have to.430                bracketing.append((p, len(stack)+1))431                lastch = ch432                p = _match_stringre(code, p, q).end()433                bracketing.append((p, len(stack)))434                continue435 436            if ch == '#':437                # consume comment and trailing newline438                bracketing.append((p, len(stack)+1))439                p = code.find('\n', p, q) + 1440                assert p > 0441                bracketing.append((p, len(stack)))442                continue443 444            assert ch == '\\'445            p = p+1     # beyond backslash446            assert p < q447            if code[p] != '\n':448                # the program is invalid, but can't complain449                lastch = ch + code[p]450            p = p+1     # beyond escaped char451 452        # end while p < q:453 454        self.lastch = lastch455        self.lastopenbracketpos = stack[-1] if stack else None456        self.stmt_bracketing = tuple(bracketing)457 458    def compute_bracket_indent(self):459        """Return number of spaces the next line should be indented.460 461        Line continuation must be C_BRACKET.462        """463        self._study2()464        assert self.continuation == C_BRACKET465        j = self.lastopenbracketpos466        code = self.code467        n = len(code)468        origi = i = code.rfind('\n', 0, j) + 1469        j = j+1     # one beyond open bracket470        # find first list item; set i to start of its line471        while j < n:472            m = _itemre(code, j)473            if m:474                j = m.end() - 1     # index of first interesting char475                extra = 0476                break477            else:478                # this line is junk; advance to next line479                i = j = code.find('\n', j) + 1480        else:481            # nothing interesting follows the bracket;482            # reproduce the bracket line's indentation + a level483            j = i = origi484            while code[j] in " \t":485                j = j+1486            extra = self.indentwidth487        return len(code[i:j].expandtabs(self.tabwidth)) + extra488 489    def get_num_lines_in_stmt(self):490        """Return number of physical lines in last stmt.491 492        The statement doesn't have to be an interesting statement.  This is493        intended to be called when continuation is C_BACKSLASH.494        """495        self._study1()496        goodlines = self.goodlines497        return goodlines[-1] - goodlines[-2]498 499    def compute_backslash_indent(self):500        """Return number of spaces the next line should be indented.501 502        Line continuation must be C_BACKSLASH.  Also assume that the new503        line is the first one following the initial line of the stmt.504        """505        self._study2()506        assert self.continuation == C_BACKSLASH507        code = self.code508        i = self.stmt_start509        while code[i] in " \t":510            i = i+1511        startpos = i512 513        # See whether the initial line starts an assignment stmt; i.e.,514        # look for an = operator515        endpos = code.find('\n', startpos) + 1516        found = level = 0517        while i < endpos:518            ch = code[i]519            if ch in "([{":520                level = level + 1521                i = i+1522            elif ch in ")]}":523                if level:524                    level = level - 1525                i = i+1526            elif ch == '"' or ch == "'":527                i = _match_stringre(code, i, endpos).end()528            elif ch == '#':529                # This line is unreachable because the # makes a comment of530                # everything after it.531                break532            elif level == 0 and ch == '=' and \533                   (i == 0 or code[i-1] not in "=<>!") and \534                   code[i+1] != '=':535                found = 1536                break537            else:538                i = i+1539 540        if found:541            # found a legit =, but it may be the last interesting542            # thing on the line543            i = i+1     # move beyond the =544            found = re.match(r"\s*\\", code[i:endpos]) is None545 546        if not found:547            # oh well ... settle for moving beyond the first chunk548            # of non-whitespace chars549            i = startpos550            while code[i] not in " \t\n":551                i = i+1552 553        return len(code[self.stmt_start:i].expandtabs(\554                                     self.tabwidth)) + 1555 556    def get_base_indent_string(self):557        """Return the leading whitespace on the initial line of the last558        interesting stmt.559        """560        self._study2()561        i, n = self.stmt_start, self.stmt_end562        j = i563        code = self.code564        while j < n and code[j] in " \t":565            j = j + 1566        return code[i:j]567 568    def is_block_opener(self):569        "Return True if the last interesting statement opens a block."570        self._study2()571        return self.lastch == ':'572 573    def is_block_closer(self):574        "Return True if the last interesting statement closes a block."575        self._study2()576        return _closere(self.code, self.stmt_start) is not None577 578    def get_last_stmt_bracketing(self):579        """Return bracketing structure of the last interesting statement.580 581        The returned tuple is in the format defined in _study2().582        """583        self._study2()584        return self.stmt_bracketing585 586 587if __name__ == '__main__':588    from unittest import main589    main('idlelib.idle_test.test_pyparse', verbosity=2)590