codekingpro/portable-devtools
114k
1#2# core.py3#4 5from collections import deque6import os7import typing8from typing import (9 Any,10 Callable,11 Generator,12 List,13 NamedTuple,14 Sequence,15 Set,16 TextIO,17 Tuple,18 Union,19 cast,20)21from abc import ABC, abstractmethod22from enum import Enum23import string24import copy25import warnings26import re27import sys28from collections.abc import Iterable29import traceback30import types31from operator import itemgetter32from functools import wraps33from threading import RLock34from pathlib import Path35 36from .util import (37 _FifoCache,38 _UnboundedCache,39 __config_flags,40 _collapse_string_to_ranges,41 _escape_regex_range_chars,42 _bslash,43 _flatten,44 LRUMemo as _LRUMemo,45 UnboundedMemo as _UnboundedMemo,46 replaced_by_pep8,47)48from .exceptions import *49from .actions import *50from .results import ParseResults, _ParseResultsWithOffset51from .unicode import pyparsing_unicode52 53_MAX_INT = sys.maxsize54str_type: Tuple[type, ...] = (str, bytes)55 56#57# Copyright (c) 2003-2022 Paul T. McGuire58#59# Permission is hereby granted, free of charge, to any person obtaining60# a copy of this software and associated documentation files (the61# "Software"), to deal in the Software without restriction, including62# without limitation the rights to use, copy, modify, merge, publish,63# distribute, sublicense, and/or sell copies of the Software, and to64# permit persons to whom the Software is furnished to do so, subject to65# the following conditions:66#67# The above copyright notice and this permission notice shall be68# included in all copies or substantial portions of the Software.69#70# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,71# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF72# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.73# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY74# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,75# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE76# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.77#78 79 80if sys.version_info >= (3, 8):81 from functools import cached_property82else:83 84 class cached_property:85 def __init__(self, func):86 self._func = func87 88 def __get__(self, instance, owner=None):89 ret = instance.__dict__[self._func.__name__] = self._func(instance)90 return ret91 92 93class __compat__(__config_flags):94 """95 A cross-version compatibility configuration for pyparsing features that will be96 released in a future version. By setting values in this configuration to True,97 those features can be enabled in prior versions for compatibility development98 and testing.99 100 - ``collect_all_And_tokens`` - flag to enable fix for Issue #63 that fixes erroneous grouping101 of results names when an :class:`And` expression is nested within an :class:`Or` or :class:`MatchFirst`;102 maintained for compatibility, but setting to ``False`` no longer restores pre-2.3.1103 behavior104 """105 106 _type_desc = "compatibility"107 108 collect_all_And_tokens = True109 110 _all_names = [__ for __ in locals() if not __.startswith("_")]111 _fixed_names = """112 collect_all_And_tokens113 """.split()114 115 116class __diag__(__config_flags):117 _type_desc = "diagnostic"118 119 warn_multiple_tokens_in_named_alternation = False120 warn_ungrouped_named_tokens_in_collection = False121 warn_name_set_on_empty_Forward = False122 warn_on_parse_using_empty_Forward = False123 warn_on_assignment_to_Forward = False124 warn_on_multiple_string_args_to_oneof = False125 warn_on_match_first_with_lshift_operator = False126 enable_debug_on_named_expressions = False127 128 _all_names = [__ for __ in locals() if not __.startswith("_")]129 _warning_names = [name for name in _all_names if name.startswith("warn")]130 _debug_names = [name for name in _all_names if name.startswith("enable_debug")]131 132 @classmethod133 def enable_all_warnings(cls) -> None:134 for name in cls._warning_names:135 cls.enable(name)136 137 138class Diagnostics(Enum):139 """140 Diagnostic configuration (all default to disabled)141 142 - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results143 name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions144 - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results145 name is defined on a containing expression with ungrouped subexpressions that also146 have results names147 - ``warn_name_set_on_empty_Forward`` - flag to enable warnings when a :class:`Forward` is defined148 with a results name, but has no contents defined149 - ``warn_on_parse_using_empty_Forward`` - flag to enable warnings when a :class:`Forward` is150 defined in a grammar but has never had an expression attached to it151 - ``warn_on_assignment_to_Forward`` - flag to enable warnings when a :class:`Forward` is defined152 but is overwritten by assigning using ``'='`` instead of ``'<<='`` or ``'<<'``153 - ``warn_on_multiple_string_args_to_oneof`` - flag to enable warnings when :class:`one_of` is154 incorrectly called with multiple str arguments155 - ``enable_debug_on_named_expressions`` - flag to auto-enable debug on all subsequent156 calls to :class:`ParserElement.set_name`157 158 Diagnostics are enabled/disabled by calling :class:`enable_diag` and :class:`disable_diag`.159 All warnings can be enabled by calling :class:`enable_all_warnings`.160 """161 162 warn_multiple_tokens_in_named_alternation = 0163 warn_ungrouped_named_tokens_in_collection = 1164 warn_name_set_on_empty_Forward = 2165 warn_on_parse_using_empty_Forward = 3166 warn_on_assignment_to_Forward = 4167 warn_on_multiple_string_args_to_oneof = 5168 warn_on_match_first_with_lshift_operator = 6169 enable_debug_on_named_expressions = 7170 171 172def enable_diag(diag_enum: Diagnostics) -> None:173 """174 Enable a global pyparsing diagnostic flag (see :class:`Diagnostics`).175 """176 __diag__.enable(diag_enum.name)177 178 179def disable_diag(diag_enum: Diagnostics) -> None:180 """181 Disable a global pyparsing diagnostic flag (see :class:`Diagnostics`).182 """183 __diag__.disable(diag_enum.name)184 185 186def enable_all_warnings() -> None:187 """188 Enable all global pyparsing diagnostic warnings (see :class:`Diagnostics`).189 """190 __diag__.enable_all_warnings()191 192 193# hide abstract class194del __config_flags195 196 197def _should_enable_warnings(198 cmd_line_warn_options: typing.Iterable[str], warn_env_var: typing.Optional[str]199) -> bool:200 enable = bool(warn_env_var)201 for warn_opt in cmd_line_warn_options:202 w_action, w_message, w_category, w_module, w_line = (warn_opt + "::::").split(203 ":"204 )[:5]205 if not w_action.lower().startswith("i") and (206 not (w_message or w_category or w_module) or w_module == "pyparsing"207 ):208 enable = True209 elif w_action.lower().startswith("i") and w_module in ("pyparsing", ""):210 enable = False211 return enable212 213 214if _should_enable_warnings(215 sys.warnoptions, os.environ.get("PYPARSINGENABLEALLWARNINGS")216):217 enable_all_warnings()218 219 220# build list of single arg builtins, that can be used as parse actions221_single_arg_builtins = {222 sum,223 len,224 sorted,225 reversed,226 list,227 tuple,228 set,229 any,230 all,231 min,232 max,233}234 235_generatorType = types.GeneratorType236ParseImplReturnType = Tuple[int, Any]237PostParseReturnType = Union[ParseResults, Sequence[ParseResults]]238ParseAction = Union[239 Callable[[], Any],240 Callable[[ParseResults], Any],241 Callable[[int, ParseResults], Any],242 Callable[[str, int, ParseResults], Any],243]244ParseCondition = Union[245 Callable[[], bool],246 Callable[[ParseResults], bool],247 Callable[[int, ParseResults], bool],248 Callable[[str, int, ParseResults], bool],249]250ParseFailAction = Callable[[str, int, "ParserElement", Exception], None]251DebugStartAction = Callable[[str, int, "ParserElement", bool], None]252DebugSuccessAction = Callable[253 [str, int, int, "ParserElement", ParseResults, bool], None254]255DebugExceptionAction = Callable[[str, int, "ParserElement", Exception, bool], None]256 257 258alphas = string.ascii_uppercase + string.ascii_lowercase259identchars = pyparsing_unicode.Latin1.identchars260identbodychars = pyparsing_unicode.Latin1.identbodychars261nums = "0123456789"262hexnums = nums + "ABCDEFabcdef"263alphanums = alphas + nums264printables = "".join([c for c in string.printable if c not in string.whitespace])265 266_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment]267 268 269def _trim_arity(func, max_limit=3):270 """decorator to trim function calls to match the arity of the target"""271 global _trim_arity_call_line272 273 if func in _single_arg_builtins:274 return lambda s, l, t: func(t)275 276 limit = 0277 found_arity = False278 279 # synthesize what would be returned by traceback.extract_stack at the call to280 # user's parse action 'func', so that we don't incur call penalty at parse time281 282 # fmt: off283 LINE_DIFF = 7284 # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND285 # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!286 _trim_arity_call_line = (_trim_arity_call_line or traceback.extract_stack(limit=2)[-1])287 pa_call_line_synth = (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)288 289 def wrapper(*args):290 nonlocal found_arity, limit291 while 1:292 try:293 ret = func(*args[limit:])294 found_arity = True295 return ret296 except TypeError as te:297 # re-raise TypeErrors if they did not come from our arity testing298 if found_arity:299 raise300 else:301 tb = te.__traceback__302 frames = traceback.extract_tb(tb, limit=2)303 frame_summary = frames[-1]304 trim_arity_type_error = (305 [frame_summary[:2]][-1][:2] == pa_call_line_synth306 )307 del tb308 309 if trim_arity_type_error:310 if limit < max_limit:311 limit += 1312 continue313 314 raise315 # fmt: on316 317 # copy func name to wrapper for sensible debug output318 # (can't use functools.wraps, since that messes with function signature)319 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)320 wrapper.__name__ = func_name321 wrapper.__doc__ = func.__doc__322 323 return wrapper324 325 326def condition_as_parse_action(327 fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False328) -> ParseAction:329 """330 Function to convert a simple predicate function that returns ``True`` or ``False``331 into a parse action. Can be used in places when a parse action is required332 and :class:`ParserElement.add_condition` cannot be used (such as when adding a condition333 to an operator level in :class:`infix_notation`).334 335 Optional keyword arguments:336 337 - ``message`` - define a custom message to be used in the raised exception338 - ``fatal`` - if True, will raise :class:`ParseFatalException` to stop parsing immediately;339 otherwise will raise :class:`ParseException`340 341 """342 msg = message if message is not None else "failed user-defined condition"343 exc_type = ParseFatalException if fatal else ParseException344 fn = _trim_arity(fn)345 346 @wraps(fn)347 def pa(s, l, t):348 if not bool(fn(s, l, t)):349 raise exc_type(s, l, msg)350 351 return pa352 353 354def _default_start_debug_action(355 instring: str, loc: int, expr: "ParserElement", cache_hit: bool = False356):357 cache_hit_str = "*" if cache_hit else ""358 print(359 (360 f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{col(loc, instring)})\n"361 f" {line(loc, instring)}\n"362 f" {' ' * (col(loc, instring) - 1)}^"363 )364 )365 366 367def _default_success_debug_action(368 instring: str,369 startloc: int,370 endloc: int,371 expr: "ParserElement",372 toks: ParseResults,373 cache_hit: bool = False,374):375 cache_hit_str = "*" if cache_hit else ""376 print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}")377 378 379def _default_exception_debug_action(380 instring: str,381 loc: int,382 expr: "ParserElement",383 exc: Exception,384 cache_hit: bool = False,385):386 cache_hit_str = "*" if cache_hit else ""387 print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}")388 389 390def null_debug_action(*args):391 """'Do-nothing' debug action, to suppress debugging output during parsing."""392 393 394class ParserElement(ABC):395 """Abstract base level parser element class."""396 397 DEFAULT_WHITE_CHARS: str = " \n\t\r"398 verbose_stacktrace: bool = False399 _literalStringClass: type = None # type: ignore[assignment]400 401 @staticmethod402 def set_default_whitespace_chars(chars: str) -> None:403 r"""404 Overrides the default whitespace chars405 406 Example::407 408 # default whitespace chars are space, <TAB> and newline409 Word(alphas)[1, ...].parse_string("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl']410 411 # change to just treat newline as significant412 ParserElement.set_default_whitespace_chars(" \t")413 Word(alphas)[1, ...].parse_string("abc def\nghi jkl") # -> ['abc', 'def']414 """415 ParserElement.DEFAULT_WHITE_CHARS = chars416 417 # update whitespace all parse expressions defined in this module418 for expr in _builtin_exprs:419 if expr.copyDefaultWhiteChars:420 expr.whiteChars = set(chars)421 422 @staticmethod423 def inline_literals_using(cls: type) -> None:424 """425 Set class to be used for inclusion of string literals into a parser.426 427 Example::428 429 # default literal class used is Literal430 integer = Word(nums)431 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")432 433 date_str.parse_string("1999/12/31") # -> ['1999', '/', '12', '/', '31']434 435 436 # change to Suppress437 ParserElement.inline_literals_using(Suppress)438 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")439 440 date_str.parse_string("1999/12/31") # -> ['1999', '12', '31']441 """442 ParserElement._literalStringClass = cls443 444 @classmethod445 def using_each(cls, seq, **class_kwargs):446 """447 Yields a sequence of class(obj, **class_kwargs) for obj in seq.448 449 Example::450 451 LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};")452 453 """454 yield from (cls(obj, **class_kwargs) for obj in seq)455 456 class DebugActions(NamedTuple):457 debug_try: typing.Optional[DebugStartAction]458 debug_match: typing.Optional[DebugSuccessAction]459 debug_fail: typing.Optional[DebugExceptionAction]460 461 def __init__(self, savelist: bool = False):462 self.parseAction: List[ParseAction] = list()463 self.failAction: typing.Optional[ParseFailAction] = None464 self.customName: str = None # type: ignore[assignment]465 self._defaultName: typing.Optional[str] = None466 self.resultsName: str = None # type: ignore[assignment]467 self.saveAsList = savelist468 self.skipWhitespace = True469 self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS)470 self.copyDefaultWhiteChars = True471 # used when checking for left-recursion472 self.mayReturnEmpty = False473 self.keepTabs = False474 self.ignoreExprs: List["ParserElement"] = list()475 self.debug = False476 self.streamlined = False477 # optimize exception handling for subclasses that don't advance parse index478 self.mayIndexError = True479 self.errmsg = ""480 # mark results names as modal (report only last) or cumulative (list all)481 self.modalResults = True482 # custom debug actions483 self.debugActions = self.DebugActions(None, None, None)484 # avoid redundant calls to preParse485 self.callPreparse = True486 self.callDuringTry = False487 self.suppress_warnings_: List[Diagnostics] = []488 489 def suppress_warning(self, warning_type: Diagnostics) -> "ParserElement":490 """491 Suppress warnings emitted for a particular diagnostic on this expression.492 493 Example::494 495 base = pp.Forward()496 base.suppress_warning(Diagnostics.warn_on_parse_using_empty_Forward)497 498 # statement would normally raise a warning, but is now suppressed499 print(base.parse_string("x"))500 501 """502 self.suppress_warnings_.append(warning_type)503 return self504 505 def visit_all(self):506 """General-purpose method to yield all expressions and sub-expressions507 in a grammar. Typically just for internal use.508 """509 to_visit = deque([self])510 seen = set()511 while to_visit:512 cur = to_visit.popleft()513 514 # guard against looping forever through recursive grammars515 if cur in seen:516 continue517 seen.add(cur)518 519 to_visit.extend(cur.recurse())520 yield cur521 522 def copy(self) -> "ParserElement":523 """524 Make a copy of this :class:`ParserElement`. Useful for defining525 different parse actions for the same parsing pattern, using copies of526 the original parse element.527 528 Example::529 530 integer = Word(nums).set_parse_action(lambda toks: int(toks[0]))531 integerK = integer.copy().add_parse_action(lambda toks: toks[0] * 1024) + Suppress("K")532 integerM = integer.copy().add_parse_action(lambda toks: toks[0] * 1024 * 1024) + Suppress("M")533 534 print((integerK | integerM | integer)[1, ...].parse_string("5K 100 640K 256M"))535 536 prints::537 538 [5120, 100, 655360, 268435456]539 540 Equivalent form of ``expr.copy()`` is just ``expr()``::541 542 integerM = integer().add_parse_action(lambda toks: toks[0] * 1024 * 1024) + Suppress("M")543 """544 cpy = copy.copy(self)545 cpy.parseAction = self.parseAction[:]546 cpy.ignoreExprs = self.ignoreExprs[:]547 if self.copyDefaultWhiteChars:548 cpy.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS)549 return cpy550 551 def set_results_name(552 self, name: str, list_all_matches: bool = False, *, listAllMatches: bool = False553 ) -> "ParserElement":554 """555 Define name for referencing matching tokens as a nested attribute556 of the returned parse results.557 558 Normally, results names are assigned as you would assign keys in a dict:559 any existing value is overwritten by later values. If it is necessary to560 keep all values captured for a particular results name, call ``set_results_name``561 with ``list_all_matches`` = True.562 563 NOTE: ``set_results_name`` returns a *copy* of the original :class:`ParserElement` object;564 this is so that the client can define a basic element, such as an565 integer, and reference it in multiple places with different names.566 567 You can also set results names using the abbreviated syntax,568 ``expr("name")`` in place of ``expr.set_results_name("name")``569 - see :class:`__call__`. If ``list_all_matches`` is required, use570 ``expr("name*")``.571 572 Example::573 574 integer = Word(nums)575 date_str = (integer.set_results_name("year") + '/'576 + integer.set_results_name("month") + '/'577 + integer.set_results_name("day"))578 579 # equivalent form:580 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")581 """582 listAllMatches = listAllMatches or list_all_matches583 return self._setResultsName(name, listAllMatches)584 585 def _setResultsName(self, name, listAllMatches=False):586 if name is None:587 return self588 newself = self.copy()589 if name.endswith("*"):590 name = name[:-1]591 listAllMatches = True592 newself.resultsName = name593 newself.modalResults = not listAllMatches594 return newself595 596 def set_break(self, break_flag: bool = True) -> "ParserElement":597 """598 Method to invoke the Python pdb debugger when this element is599 about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to600 disable.601 """602 if break_flag:603 _parseMethod = self._parse604 605 def breaker(instring, loc, doActions=True, callPreParse=True):606 import pdb607 608 # this call to pdb.set_trace() is intentional, not a checkin error609 pdb.set_trace()610 return _parseMethod(instring, loc, doActions, callPreParse)611 612 breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined]613 self._parse = breaker # type: ignore [assignment]614 elif hasattr(self._parse, "_originalParseMethod"):615 self._parse = self._parse._originalParseMethod # type: ignore [attr-defined, assignment]616 return self617 618 def set_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement":619 """620 Define one or more actions to perform when successfully matching parse element definition.621 622 Parse actions can be called to perform data conversions, do extra validation,623 update external data structures, or enhance or replace the parsed tokens.624 Each parse action ``fn`` is a callable method with 0-3 arguments, called as625 ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where:626 627 - ``s`` = the original string being parsed (see note below)628 - ``loc`` = the location of the matching substring629 - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object630 631 The parsed tokens are passed to the parse action as ParseResults. They can be632 modified in place using list-style append, extend, and pop operations to update633 the parsed list elements; and with dictionary-style item set and del operations634 to add, update, or remove any named results. If the tokens are modified in place,635 it is not necessary to return them with a return statement.636 637 Parse actions can also completely replace the given tokens, with another ``ParseResults``638 object, or with some entirely different object (common for parse actions that perform data639 conversions). A convenient way to build a new parse result is to define the values640 using a dict, and then create the return value using :class:`ParseResults.from_dict`.641 642 If None is passed as the ``fn`` parse action, all previously added parse actions for this643 expression are cleared.644 645 Optional keyword arguments:646 647 - ``call_during_try`` = (default= ``False``) indicate if parse action should be run during648 lookaheads and alternate testing. For parse actions that have side effects, it is649 important to only call the parse action once it is determined that it is being650 called as part of a successful parse. For parse actions that perform additional651 validation, then call_during_try should be passed as True, so that the validation652 code is included in the preliminary "try" parses.653 654 Note: the default parsing behavior is to expand tabs in the input string655 before starting the parsing process. See :class:`parse_string` for more656 information on parsing strings containing ``<TAB>`` s, and suggested657 methods to maintain a consistent view of the parsed string, the parse658 location, and line and column positions within the parsed string.659 660 Example::661 662 # parse dates in the form YYYY/MM/DD663 664 # use parse action to convert toks from str to int at parse time665 def convert_to_int(toks):666 return int(toks[0])667 668 # use a parse action to verify that the date is a valid date669 def is_valid_date(instring, loc, toks):670 from datetime import date671 year, month, day = toks[::2]672 try:673 date(year, month, day)674 except ValueError:675 raise ParseException(instring, loc, "invalid date given")676 677 integer = Word(nums)678 date_str = integer + '/' + integer + '/' + integer679 680 # add parse actions681 integer.set_parse_action(convert_to_int)682 date_str.set_parse_action(is_valid_date)683 684 # note that integer fields are now ints, not strings685 date_str.run_tests('''686 # successful parse - note that integer fields were converted to ints687 1999/12/31688 689 # fail - invalid date690 1999/13/31691 ''')692 """693 if list(fns) == [None]:694 self.parseAction = []695 return self696 697 if not all(callable(fn) for fn in fns):698 raise TypeError("parse actions must be callable")699 self.parseAction = [_trim_arity(fn) for fn in fns]700 self.callDuringTry = kwargs.get(701 "call_during_try", kwargs.get("callDuringTry", False)702 )703 704 return self705 706 def add_parse_action(self, *fns: ParseAction, **kwargs) -> "ParserElement":707 """708 Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`.709 710 See examples in :class:`copy`.711 """712 self.parseAction += [_trim_arity(fn) for fn in fns]713 self.callDuringTry = self.callDuringTry or kwargs.get(714 "call_during_try", kwargs.get("callDuringTry", False)715 )716 return self717 718 def add_condition(self, *fns: ParseCondition, **kwargs) -> "ParserElement":719 """Add a boolean predicate function to expression's list of parse actions. See720 :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``,721 functions passed to ``add_condition`` need to return boolean success/fail of the condition.722 723 Optional keyword arguments:724 725 - ``message`` = define a custom message to be used in the raised exception726 - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise727 ParseException728 - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls,729 default=False730 731 Example::732 733 integer = Word(nums).set_parse_action(lambda toks: int(toks[0]))734 year_int = integer.copy()735 year_int.add_condition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")736 date_str = year_int + '/' + integer + '/' + integer737 738 result = date_str.parse_string("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0),739 (line:1, col:1)740 """741 for fn in fns:742 self.parseAction.append(743 condition_as_parse_action(744 fn,745 message=str(kwargs.get("message")),746 fatal=bool(kwargs.get("fatal", False)),747 )748 )749 750 self.callDuringTry = self.callDuringTry or kwargs.get(751 "call_during_try", kwargs.get("callDuringTry", False)752 )753 return self754 755 def set_fail_action(self, fn: ParseFailAction) -> "ParserElement":756 """757 Define action to perform if parsing fails at this expression.758 Fail acton fn is a callable function that takes the arguments759 ``fn(s, loc, expr, err)`` where:760 761 - ``s`` = string being parsed762 - ``loc`` = location where expression match was attempted and failed763 - ``expr`` = the parse expression that failed764 - ``err`` = the exception thrown765 766 The function returns no value. It may throw :class:`ParseFatalException`767 if it is desired to stop parsing immediately."""768 self.failAction = fn769 return self770 771 def _skipIgnorables(self, instring: str, loc: int) -> int:772 if not self.ignoreExprs:773 return loc774 exprsFound = True775 ignore_expr_fns = [e._parse for e in self.ignoreExprs]776 last_loc = loc777 while exprsFound:778 exprsFound = False779 for ignore_fn in ignore_expr_fns:780 try:781 while 1:782 loc, dummy = ignore_fn(instring, loc)783 exprsFound = True784 except ParseException:785 pass786 # check if all ignore exprs matched but didn't actually advance the parse location787 if loc == last_loc:788 break789 last_loc = loc790 return loc791 792 def preParse(self, instring: str, loc: int) -> int:793 if self.ignoreExprs:794 loc = self._skipIgnorables(instring, loc)795 796 if self.skipWhitespace:797 instrlen = len(instring)798 white_chars = self.whiteChars799 while loc < instrlen and instring[loc] in white_chars:800 loc += 1801 802 return loc803 804 def parseImpl(self, instring, loc, doActions=True):805 return loc, []806 807 def postParse(self, instring, loc, tokenlist):808 return tokenlist809 810 # @profile811 def _parseNoCache(812 self, instring, loc, doActions=True, callPreParse=True813 ) -> Tuple[int, ParseResults]:814 TRY, MATCH, FAIL = 0, 1, 2815 debugging = self.debug # and doActions)816 len_instring = len(instring)817 818 if debugging or self.failAction:819 # print("Match {} at loc {}({}, {})".format(self, loc, lineno(loc, instring), col(loc, instring)))820 try:821 if callPreParse and self.callPreparse:822 pre_loc = self.preParse(instring, loc)823 else:824 pre_loc = loc825 tokens_start = pre_loc826 if self.debugActions.debug_try:827 self.debugActions.debug_try(instring, tokens_start, self, False)828 if self.mayIndexError or pre_loc >= len_instring:829 try:830 loc, tokens = self.parseImpl(instring, pre_loc, doActions)831 except IndexError:832 raise ParseException(instring, len_instring, self.errmsg, self)833 else:834 loc, tokens = self.parseImpl(instring, pre_loc, doActions)835 except Exception as err:836 # print("Exception raised:", err)837 if self.debugActions.debug_fail:838 self.debugActions.debug_fail(839 instring, tokens_start, self, err, False840 )841 if self.failAction:842 self.failAction(instring, tokens_start, self, err)843 raise844 else:845 if callPreParse and self.callPreparse:846 pre_loc = self.preParse(instring, loc)847 else:848 pre_loc = loc849 tokens_start = pre_loc850 if self.mayIndexError or pre_loc >= len_instring:851 try:852 loc, tokens = self.parseImpl(instring, pre_loc, doActions)853 except IndexError:854 raise ParseException(instring, len_instring, self.errmsg, self)855 else:856 loc, tokens = self.parseImpl(instring, pre_loc, doActions)857 858 tokens = self.postParse(instring, loc, tokens)859 860 ret_tokens = ParseResults(861 tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults862 )863 if self.parseAction and (doActions or self.callDuringTry):864 if debugging:865 try:866 for fn in self.parseAction:867 try:868 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]869 except IndexError as parse_action_exc:870 exc = ParseException("exception raised in parse action")871 raise exc from parse_action_exc872 873 if tokens is not None and tokens is not ret_tokens:874 ret_tokens = ParseResults(875 tokens,876 self.resultsName,877 asList=self.saveAsList878 and isinstance(tokens, (ParseResults, list)),879 modal=self.modalResults,880 )881 except Exception as err:882 # print "Exception raised in user parse action:", err883 if self.debugActions.debug_fail:884 self.debugActions.debug_fail(885 instring, tokens_start, self, err, False886 )887 raise888 else:889 for fn in self.parseAction:890 try:891 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]892 except IndexError as parse_action_exc:893 exc = ParseException("exception raised in parse action")894 raise exc from parse_action_exc895 896 if tokens is not None and tokens is not ret_tokens:897 ret_tokens = ParseResults(898 tokens,899 self.resultsName,900 asList=self.saveAsList901 and isinstance(tokens, (ParseResults, list)),902 modal=self.modalResults,903 )904 if debugging:905 # print("Matched", self, "->", ret_tokens.as_list())906 if self.debugActions.debug_match:907 self.debugActions.debug_match(908 instring, tokens_start, loc, self, ret_tokens, False909 )910 911 return loc, ret_tokens912 913 def try_parse(914 self,915 instring: str,916 loc: int,917 *,918 raise_fatal: bool = False,919 do_actions: bool = False,920 ) -> int:921 try:922 return self._parse(instring, loc, doActions=do_actions)[0]923 except ParseFatalException:924 if raise_fatal:925 raise926 raise ParseException(instring, loc, self.errmsg, self)927 928 def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool:929 try:930 self.try_parse(instring, loc, do_actions=do_actions)931 except (ParseException, IndexError):932 return False933 else:934 return True935 936 # cache for left-recursion in Forward references937 recursion_lock = RLock()938 recursion_memos: typing.Dict[939 Tuple[int, "Forward", bool], Tuple[int, Union[ParseResults, Exception]]940 ] = {}941 942 class _CacheType(dict):943 """944 class to help type checking945 """946 947 not_in_cache: bool948 949 def get(self, *args): ...950 951 def set(self, *args): ...952 953 # argument cache for optimizing repeated calls when backtracking through recursive expressions954 packrat_cache = (955 _CacheType()956 ) # set later by enable_packrat(); this is here so that reset_cache() doesn't fail957 packrat_cache_lock = RLock()958 packrat_cache_stats = [0, 0]959 960 # this method gets repeatedly called during backtracking with the same arguments -961 # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression962 def _parseCache(963 self, instring, loc, doActions=True, callPreParse=True964 ) -> Tuple[int, ParseResults]:965 HIT, MISS = 0, 1966 TRY, MATCH, FAIL = 0, 1, 2967 lookup = (self, instring, loc, callPreParse, doActions)968 with ParserElement.packrat_cache_lock:969 cache = ParserElement.packrat_cache970 value = cache.get(lookup)971 if value is cache.not_in_cache:972 ParserElement.packrat_cache_stats[MISS] += 1973 try:974 value = self._parseNoCache(instring, loc, doActions, callPreParse)975 except ParseBaseException as pe:976 # cache a copy of the exception, without the traceback977 cache.set(lookup, pe.__class__(*pe.args))978 raise979 else:980 cache.set(lookup, (value[0], value[1].copy(), loc))981 return value982 else:983 ParserElement.packrat_cache_stats[HIT] += 1984 if self.debug and self.debugActions.debug_try:985 try:986 self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg]987 except TypeError:988 pass989 if isinstance(value, Exception):990 if self.debug and self.debugActions.debug_fail:991 try:992 self.debugActions.debug_fail(993 instring, loc, self, value, cache_hit=True # type: ignore [call-arg]994 )995 except TypeError:996 pass997 raise value998 999 value = cast(Tuple[int, ParseResults, int], value)1000 loc_, result, endloc = value[0], value[1].copy(), value[2]1001 if self.debug and self.debugActions.debug_match:1002 try:1003 self.debugActions.debug_match(1004 instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg]1005 )1006 except TypeError:1007 pass1008 1009 return loc_, result1010 1011 _parse = _parseNoCache1012 1013 @staticmethod1014 def reset_cache() -> None:1015 ParserElement.packrat_cache.clear()1016 ParserElement.packrat_cache_stats[:] = [0] * len(1017 ParserElement.packrat_cache_stats1018 )1019 ParserElement.recursion_memos.clear()1020 1021 _packratEnabled = False1022 _left_recursion_enabled = False1023 1024 @staticmethod1025 def disable_memoization() -> None:1026 """1027 Disables active Packrat or Left Recursion parsing and their memoization1028 1029 This method also works if neither Packrat nor Left Recursion are enabled.1030 This makes it safe to call before activating Packrat nor Left Recursion1031 to clear any previous settings.1032 """1033 ParserElement.reset_cache()1034 ParserElement._left_recursion_enabled = False1035 ParserElement._packratEnabled = False1036 ParserElement._parse = ParserElement._parseNoCache1037 1038 @staticmethod1039 def enable_left_recursion(1040 cache_size_limit: typing.Optional[int] = None, *, force=False1041 ) -> None:1042 """1043 Enables "bounded recursion" parsing, which allows for both direct and indirect1044 left-recursion. During parsing, left-recursive :class:`Forward` elements are1045 repeatedly matched with a fixed recursion depth that is gradually increased1046 until finding the longest match.1047 1048 Example::1049 1050 import pyparsing as pp1051 pp.ParserElement.enable_left_recursion()1052 1053 E = pp.Forward("E")1054 num = pp.Word(pp.nums)1055 # match `num`, or `num '+' num`, or `num '+' num '+' num`, ...1056 E <<= E + '+' - num | num1057 1058 print(E.parse_string("1+2+3"))1059 1060 Recursion search naturally memoizes matches of ``Forward`` elements and may1061 thus skip reevaluation of parse actions during backtracking. This may break1062 programs with parse actions which rely on strict ordering of side-effects.1063 1064 Parameters:1065 1066 - ``cache_size_limit`` - (default=``None``) - memoize at most this many1067 ``Forward`` elements during matching; if ``None`` (the default),1068 memoize all ``Forward`` elements.1069 1070 Bounded Recursion parsing works similar but not identical to Packrat parsing,1071 thus the two cannot be used together. Use ``force=True`` to disable any1072 previous, conflicting settings.1073 """1074 if force:1075 ParserElement.disable_memoization()1076 elif ParserElement._packratEnabled:1077 raise RuntimeError("Packrat and Bounded Recursion are not compatible")1078 if cache_size_limit is None:1079 ParserElement.recursion_memos = _UnboundedMemo() # type: ignore[assignment]1080 elif cache_size_limit > 0:1081 ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment]1082 else:1083 raise NotImplementedError(f"Memo size of {cache_size_limit}")1084 ParserElement._left_recursion_enabled = True1085 1086 @staticmethod1087 def enable_packrat(1088 cache_size_limit: Union[int, None] = 128, *, force: bool = False1089 ) -> None:1090 """1091 Enables "packrat" parsing, which adds memoizing to the parsing logic.1092 Repeated parse attempts at the same string location (which happens1093 often in many complex grammars) can immediately return a cached value,1094 instead of re-executing parsing/validating code. Memoizing is done of1095 both valid results and parsing exceptions.1096 1097 Parameters:1098 1099 - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided1100 will limit the size of the packrat cache; if None is passed, then1101 the cache size will be unbounded; if 0 is passed, the cache will1102 be effectively disabled.1103 1104 This speedup may break existing programs that use parse actions that1105 have side-effects. For this reason, packrat parsing is disabled when1106 you first import pyparsing. To activate the packrat feature, your1107 program must call the class method :class:`ParserElement.enable_packrat`.1108 For best results, call ``enable_packrat()`` immediately after1109 importing pyparsing.1110 1111 Example::1112 1113 import pyparsing1114 pyparsing.ParserElement.enable_packrat()1115 1116 Packrat parsing works similar but not identical to Bounded Recursion parsing,1117 thus the two cannot be used together. Use ``force=True`` to disable any1118 previous, conflicting settings.1119 """1120 if force:1121 ParserElement.disable_memoization()1122 elif ParserElement._left_recursion_enabled:1123 raise RuntimeError("Packrat and Bounded Recursion are not compatible")1124 1125 if ParserElement._packratEnabled:1126 return1127 1128 ParserElement._packratEnabled = True1129 if cache_size_limit is None:1130 ParserElement.packrat_cache = _UnboundedCache()1131 else:1132 ParserElement.packrat_cache = _FifoCache(cache_size_limit) # type: ignore[assignment]1133 ParserElement._parse = ParserElement._parseCache1134 1135 def parse_string(1136 self, instring: str, parse_all: bool = False, *, parseAll: bool = False1137 ) -> ParseResults:1138 """1139 Parse a string with respect to the parser definition. This function is intended as the primary interface to the1140 client code.1141 1142 :param instring: The input string to be parsed.1143 :param parse_all: If set, the entire input string must match the grammar.1144 :param parseAll: retained for pre-PEP8 compatibility, will be removed in a future release.1145 :raises ParseException: Raised if ``parse_all`` is set and the input string does not match the whole grammar.1146 :returns: the parsed data as a :class:`ParseResults` object, which may be accessed as a `list`, a `dict`, or1147 an object with attributes if the given parser includes results names.1148 1149 If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This1150 is also equivalent to ending the grammar with :class:`StringEnd`\\ ().1151 1152 To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are1153 converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string1154 contains tabs and the grammar uses parse actions that use the ``loc`` argument to index into the string1155 being parsed, one can ensure a consistent view of the input string by doing one of the following:1156 1157 - calling ``parse_with_tabs`` on your grammar before calling ``parse_string`` (see :class:`parse_with_tabs`),1158 - define your parse action using the full ``(s,loc,toks)`` signature, and reference the input string using the1159 parse action's ``s`` argument, or1160 - explicitly expand the tabs in your input string before calling ``parse_string``.1161 1162 Examples:1163 1164 By default, partial matches are OK.1165 1166 >>> res = Word('a').parse_string('aaaaabaaa')1167 >>> print(res)1168 ['aaaaa']1169 1170 The parsing behavior varies by the inheriting class of this abstract class. Please refer to the children1171 directly to see more examples.1172 1173 It raises an exception if parse_all flag is set and instring does not match the whole grammar.1174 1175 >>> res = Word('a').parse_string('aaaaabaaa', parse_all=True)1176 Traceback (most recent call last):1177 ...1178 pyparsing.ParseException: Expected end of text, found 'b' (at char 5), (line:1, col:6)1179 """1180 parseAll = parse_all or parseAll1181 1182 ParserElement.reset_cache()1183 if not self.streamlined:1184 self.streamline()1185 for e in self.ignoreExprs:1186 e.streamline()1187 if not self.keepTabs:1188 instring = instring.expandtabs()1189 try:1190 loc, tokens = self._parse(instring, 0)1191 if parseAll:1192 loc = self.preParse(instring, loc)1193 se = Empty() + StringEnd()1194 se._parse(instring, loc)1195 except ParseBaseException as exc:1196 if ParserElement.verbose_stacktrace:1197 raise1198 else:1199 # catch and re-raise exception from here, clearing out pyparsing internal stack trace1200 raise exc.with_traceback(None)