codekingpro/portable-devtools
114k
1 2from __future__ import annotations3 4# Scanner produces tokens of the following types:5# STREAM-START6# STREAM-END7# DIRECTIVE(name, value)8# DOCUMENT-START9# DOCUMENT-END10# BLOCK-SEQUENCE-START11# BLOCK-MAPPING-START12# BLOCK-END13# FLOW-SEQUENCE-START14# FLOW-MAPPING-START15# FLOW-SEQUENCE-END16# FLOW-MAPPING-END17# BLOCK-ENTRY18# FLOW-ENTRY19# KEY20# VALUE21# ALIAS(value)22# ANCHOR(value)23# TAG(value)24# SCALAR(value, plain, style)25#26# RoundTripScanner27# COMMENT(value)28#29# Read comments in the Scanner code for more details.30#31 32from ruamel.yaml.error import MarkedYAMLError33import ruamel.yaml.tokens as tokens34from ruamel.yaml.docinfo import Version # NOQA35from ruamel.yaml.compat import check_anchorname_char, _debug, nprint, nprintf # NOQA36 37if False: # MYPY38 from typing import Any, Dict, Optional, List, Union, Text, Tuple # NOQA39 40__all__ = ['Scanner', 'RoundTripScanner', 'ScannerError']41 42 43_THE_END = '\n\0\r\x85\u2028\u2029'44_THE_END_SPACE_TAB = ' \n\0\t\r\x85\u2028\u2029'45_SPACE_TAB = ' \t'46 47 48if _debug != 0:49 def xprintf(*args: Any, **kw: Any) -> Any:50 return nprintf(*args, **kw)51 52 53class ScannerError(MarkedYAMLError):54 pass55 56 57class SimpleKey:58 # See below simple keys treatment.59 60 def __init__(61 self, token_number: Any, required: Any, index: int, line: int, column: int, mark: Any,62 ) -> None:63 self.token_number = token_number64 self.required = required65 self.index = index66 self.line = line67 self.column = column68 self.mark = mark69 70 71class Scanner:72 def __init__(self, loader: Any = None) -> None:73 """Initialize the scanner."""74 # It is assumed that Scanner and Reader will have a common descendant.75 # Reader do the dirty work of checking for BOM and converting the76 # input data to Unicode. It also adds NUL to the end.77 #78 # Reader supports the following methods79 # self.peek(i=0) # peek the next i-th character80 # self.prefix(l=1) # peek the next l characters81 # self.forward(l=1) # read the next l characters and move the pointer82 83 self.loader = loader84 if self.loader is not None and getattr(self.loader, '_scanner', None) is None:85 self.loader._scanner = self86 self.reset_scanner()87 self.first_time = False88 89 @property90 def flow_level(self) -> int:91 return len(self.flow_context)92 93 def reset_scanner(self) -> None:94 # Had we reached the end of the stream?95 self.done = False96 97 # flow_context is an expanding/shrinking list consisting of '{' and '['98 # for each unclosed flow context. If empty list that means block context99 self.flow_context: List[Text] = []100 101 # List of processed tokens that are not yet emitted.102 self.tokens: List[Any] = []103 104 # Add the STREAM-START token.105 self.fetch_stream_start()106 107 # Number of tokens that were emitted through the `get_token` method.108 self.tokens_taken = 0109 110 # The current indentation level.111 self.indent = -1112 113 # Past indentation levels.114 self.indents: List[int] = []115 116 # Variables related to simple keys treatment.117 118 # A simple key is a key that is not denoted by the '?' indicator.119 # Example of simple keys:120 # ---121 # block simple key: value122 # ? not a simple key:123 # : { flow simple key: value }124 # We emit the KEY token before all keys, so when we find a potential125 # simple key, we try to locate the corresponding ':' indicator.126 # Simple keys should be limited to a single line and 1024 characters.127 128 # Can a simple key start at the current position? A simple key may129 # start:130 # - at the beginning of the line, not counting indentation spaces131 # (in block context),132 # - after '{', '[', ',' (in the flow context),133 # - after '?', ':', '-' (in the block context).134 # In the block context, this flag also signifies if a block collection135 # may start at the current position.136 self.allow_simple_key = True137 138 # Keep track of possible simple keys. This is a dictionary. The key139 # is `flow_level`; there can be no more that one possible simple key140 # for each level. The value is a SimpleKey record:141 # (token_number, required, index, line, column, mark)142 # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),143 # '[', or '{' tokens.144 self.possible_simple_keys: Dict[Any, Any] = {}145 self.yaml_version: Any = None146 self.tag_directives: List[Tuple[Any, Any]] = []147 148 @property149 def reader(self) -> Any:150 try:151 return self._scanner_reader # type: ignore152 except AttributeError:153 if hasattr(self.loader, 'typ'):154 self._scanner_reader = self.loader.reader155 else:156 self._scanner_reader = self.loader._reader157 return self._scanner_reader158 159 @property160 def scanner_processing_version(self) -> Any: # prefix until un-composited161 if hasattr(self.loader, 'typ'):162 return self.loader.resolver.processing_version163 return self.loader.processing_version164 165 # Public methods.166 167 def check_token(self, *choices: Any) -> bool:168 # Check if the next token is one of the given types.169 while self.need_more_tokens():170 self.fetch_more_tokens()171 if len(self.tokens) > 0:172 if not choices:173 return True174 for choice in choices:175 if isinstance(self.tokens[0], choice):176 return True177 return False178 179 def peek_token(self) -> Any:180 # Return the next token, but do not delete if from the queue.181 while self.need_more_tokens():182 self.fetch_more_tokens()183 if len(self.tokens) > 0:184 return self.tokens[0]185 186 def get_token(self) -> Any:187 # Return the next token.188 while self.need_more_tokens():189 self.fetch_more_tokens()190 if len(self.tokens) > 0:191 self.tokens_taken += 1192 return self.tokens.pop(0)193 194 # Private methods.195 196 def need_more_tokens(self) -> bool:197 if self.done:198 return False199 if len(self.tokens) == 0:200 return True201 # The current token may be a potential simple key, so we202 # need to look further.203 self.stale_possible_simple_keys()204 if self.next_possible_simple_key() == self.tokens_taken:205 return True206 return False207 208 def fetch_comment(self, comment: Any) -> None:209 raise NotImplementedError210 211 def fetch_more_tokens(self) -> Any:212 # Eat whitespaces and comments until we reach the next token.213 comment = self.scan_to_next_token()214 if comment is not None: # never happens for base scanner215 return self.fetch_comment(comment)216 # Remove obsolete possible simple keys.217 self.stale_possible_simple_keys()218 219 # Compare the current indentation and column. It may add some tokens220 # and decrease the current indentation level.221 self.unwind_indent(self.reader.column)222 223 # Peek the next character.224 ch = self.reader.peek()225 226 # Is it the end of stream?227 if ch == '\0':228 return self.fetch_stream_end()229 230 # Is it a directive?231 if ch == '%' and self.check_directive():232 return self.fetch_directive()233 234 # Is it the document start?235 if ch == '-' and self.check_document_start():236 return self.fetch_document_start()237 238 # Is it the document end?239 if ch == '.' and self.check_document_end():240 return self.fetch_document_end()241 242 # TODO: support for BOM within a stream.243 # if ch == '\uFEFF':244 # return self.fetch_bom() <-- issue BOMToken245 246 # Note: the order of the following checks is NOT significant.247 248 # Is it the flow sequence start indicator?249 if ch == '[':250 return self.fetch_flow_sequence_start()251 252 # Is it the flow mapping start indicator?253 if ch == '{':254 return self.fetch_flow_mapping_start()255 256 # Is it the flow sequence end indicator?257 if ch == ']':258 return self.fetch_flow_sequence_end()259 260 # Is it the flow mapping end indicator?261 if ch == '}':262 return self.fetch_flow_mapping_end()263 264 # Is it the flow entry indicator?265 if ch == ',':266 return self.fetch_flow_entry()267 268 # Is it the block entry indicator?269 if ch == '-' and self.check_block_entry():270 return self.fetch_block_entry()271 272 # Is it the key indicator?273 if ch == '?' and self.check_key():274 return self.fetch_key()275 276 # Is it the value indicator?277 if ch == ':' and self.check_value():278 return self.fetch_value()279 280 # Is it an alias?281 if ch == '*':282 return self.fetch_alias()283 284 # Is it an anchor?285 if ch == '&':286 return self.fetch_anchor()287 288 # Is it a tag?289 if ch == '!':290 return self.fetch_tag()291 292 # Is it a literal scalar?293 if ch == '|' and not self.flow_level:294 return self.fetch_literal()295 296 # Is it a folded scalar?297 if ch == '>' and not self.flow_level:298 return self.fetch_folded()299 300 # Is it a single quoted scalar?301 if ch == "'":302 return self.fetch_single()303 304 # Is it a double quoted scalar?305 if ch == '"':306 return self.fetch_double()307 308 # It must be a plain scalar then.309 if self.check_plain():310 return self.fetch_plain()311 312 # No? It's an error. Let's produce a nice error message.313 raise ScannerError(314 'while scanning for the next token',315 None,316 f'found character {ch!r} that cannot start any token',317 self.reader.get_mark(),318 )319 320 # Simple keys treatment.321 322 def next_possible_simple_key(self) -> Any:323 # Return the number of the nearest possible simple key. Actually we324 # don't need to loop through the whole dictionary. We may replace it325 # with the following code:326 # if not self.possible_simple_keys:327 # return None328 # return self.possible_simple_keys[329 # min(self.possible_simple_keys.keys())].token_number330 min_token_number = None331 for level in self.possible_simple_keys:332 key = self.possible_simple_keys[level]333 if min_token_number is None or key.token_number < min_token_number:334 min_token_number = key.token_number335 return min_token_number336 337 def stale_possible_simple_keys(self) -> None:338 # Remove entries that are no longer possible simple keys. According to339 # the YAML specification, simple keys340 # - should be limited to a single line,341 # - should be no longer than 1024 characters.342 # Disabling this procedure will allow simple keys of any length and343 # height (may cause problems if indentation is broken though).344 for level in list(self.possible_simple_keys):345 key = self.possible_simple_keys[level]346 if key.line != self.reader.line or self.reader.index - key.index > 1024:347 if key.required:348 raise ScannerError(349 'while scanning a simple key',350 key.mark,351 "could not find expected ':'",352 self.reader.get_mark(),353 )354 del self.possible_simple_keys[level]355 356 def save_possible_simple_key(self) -> None:357 # The next token may start a simple key. We check if it's possible358 # and save its position. This function is called for359 # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.360 361 # Check if a simple key is required at the current position.362 required = not self.flow_level and self.indent == self.reader.column363 364 # The next token might be a simple key. Let's save it's number and365 # position.366 if self.allow_simple_key:367 self.remove_possible_simple_key()368 token_number = self.tokens_taken + len(self.tokens)369 key = SimpleKey(370 token_number,371 required,372 self.reader.index,373 self.reader.line,374 self.reader.column,375 self.reader.get_mark(),376 )377 self.possible_simple_keys[self.flow_level] = key378 379 def remove_possible_simple_key(self) -> None:380 # Remove the saved possible key position at the current flow level.381 if self.flow_level in self.possible_simple_keys:382 key = self.possible_simple_keys[self.flow_level]383 384 if key.required:385 raise ScannerError(386 'while scanning a simple key',387 key.mark,388 "could not find expected ':'",389 self.reader.get_mark(),390 )391 392 del self.possible_simple_keys[self.flow_level]393 394 # Indentation functions.395 396 def unwind_indent(self, column: Any) -> None:397 # In flow context, tokens should respect indentation.398 # Actually the condition should be `self.indent >= column` according to399 # the spec. But this condition will prohibit intuitively correct400 # constructions such as401 # key : {402 # }403 # ####404 # if self.flow_level and self.indent > column:405 # raise ScannerError(None, None,406 # "invalid intendation or unclosed '[' or '{'",407 # self.reader.get_mark())408 409 # In the flow context, indentation is ignored. We make the scanner less410 # restrictive then specification requires.411 if bool(self.flow_level):412 return413 414 # In block context, we may need to issue the BLOCK-END tokens.415 while self.indent > column:416 mark = self.reader.get_mark()417 self.indent = self.indents.pop()418 self.tokens.append(tokens.BlockEndToken(mark, mark))419 420 def add_indent(self, column: int) -> bool:421 # Check if we need to increase indentation.422 if self.indent < column:423 self.indents.append(self.indent)424 self.indent = column425 return True426 return False427 428 # Fetchers.429 430 def fetch_stream_start(self) -> None:431 # We always add STREAM-START as the first token and STREAM-END as the432 # last token.433 # Read the token.434 mark = self.reader.get_mark()435 # Add STREAM-START.436 self.tokens.append(tokens.StreamStartToken(mark, mark, encoding=self.reader.encoding))437 438 def fetch_stream_end(self) -> None:439 # Set the current intendation to -1.440 self.unwind_indent(-1)441 # Reset simple keys.442 self.remove_possible_simple_key()443 self.allow_simple_key = False444 self.possible_simple_keys = {}445 # Read the token.446 mark = self.reader.get_mark()447 # Add STREAM-END.448 self.tokens.append(tokens.StreamEndToken(mark, mark))449 # The steam is finished.450 self.done = True451 452 def fetch_directive(self) -> None:453 # Set the current intendation to -1.454 self.unwind_indent(-1)455 456 # Reset simple keys.457 self.remove_possible_simple_key()458 self.allow_simple_key = False459 460 # Scan and add DIRECTIVE.461 self.tokens.append(self.scan_directive())462 463 def fetch_document_start(self) -> None:464 self.fetch_document_indicator(tokens.DocumentStartToken)465 466 def fetch_document_end(self) -> None:467 self.fetch_document_indicator(tokens.DocumentEndToken)468 469 def fetch_document_indicator(self, TokenClass: Any) -> None:470 # Set the current intendation to -1.471 self.unwind_indent(-1)472 473 # Reset simple keys. Note that there could not be a block collection474 # after '---'.475 self.remove_possible_simple_key()476 self.allow_simple_key = False477 478 # Add DOCUMENT-START or DOCUMENT-END.479 start_mark = self.reader.get_mark()480 self.reader.forward(3)481 end_mark = self.reader.get_mark()482 self.tokens.append(TokenClass(start_mark, end_mark))483 484 def fetch_flow_sequence_start(self) -> None:485 self.fetch_flow_collection_start(tokens.FlowSequenceStartToken, to_push='[')486 487 def fetch_flow_mapping_start(self) -> None:488 self.fetch_flow_collection_start(tokens.FlowMappingStartToken, to_push='{')489 490 def fetch_flow_collection_start(self, TokenClass: Any, to_push: Text) -> None:491 # '[' and '{' may start a simple key.492 self.save_possible_simple_key()493 # Increase the flow level.494 self.flow_context.append(to_push)495 # Simple keys are allowed after '[' and '{'.496 self.allow_simple_key = True497 # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.498 start_mark = self.reader.get_mark()499 self.reader.forward()500 end_mark = self.reader.get_mark()501 self.tokens.append(TokenClass(start_mark, end_mark))502 503 def fetch_flow_sequence_end(self) -> None:504 self.fetch_flow_collection_end(tokens.FlowSequenceEndToken)505 506 def fetch_flow_mapping_end(self) -> None:507 self.fetch_flow_collection_end(tokens.FlowMappingEndToken)508 509 def fetch_flow_collection_end(self, TokenClass: Any) -> None:510 # Reset possible simple key on the current level.511 self.remove_possible_simple_key()512 # Decrease the flow level.513 try:514 popped = self.flow_context.pop() # NOQA515 except IndexError:516 # We must not be in a list or object.517 # Defer error handling to the parser.518 pass519 # No simple keys after ']' or '}'.520 self.allow_simple_key = False521 # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.522 start_mark = self.reader.get_mark()523 self.reader.forward()524 end_mark = self.reader.get_mark()525 self.tokens.append(TokenClass(start_mark, end_mark))526 527 def fetch_flow_entry(self) -> None:528 # Simple keys are allowed after ','.529 self.allow_simple_key = True530 # Reset possible simple key on the current level.531 self.remove_possible_simple_key()532 # Add FLOW-ENTRY.533 start_mark = self.reader.get_mark()534 self.reader.forward()535 end_mark = self.reader.get_mark()536 self.tokens.append(tokens.FlowEntryToken(start_mark, end_mark))537 538 def fetch_block_entry(self) -> None:539 # Block context needs additional checks.540 if not self.flow_level:541 # Are we allowed to start a new entry?542 if not self.allow_simple_key:543 raise ScannerError(544 None,545 None,546 'sequence entries are not allowed here',547 self.reader.get_mark(),548 )549 # We may need to add BLOCK-SEQUENCE-START.550 if self.add_indent(self.reader.column):551 mark = self.reader.get_mark()552 self.tokens.append(tokens.BlockSequenceStartToken(mark, mark))553 # It's an error for the block entry to occur in the flow context,554 # but we let the parser detect this.555 else:556 pass557 # Simple keys are allowed after '-'.558 self.allow_simple_key = True559 # Reset possible simple key on the current level.560 self.remove_possible_simple_key()561 562 # Add BLOCK-ENTRY.563 start_mark = self.reader.get_mark()564 self.reader.forward()565 end_mark = self.reader.get_mark()566 self.tokens.append(tokens.BlockEntryToken(start_mark, end_mark))567 568 def fetch_key(self) -> None:569 # Block context needs additional checks.570 if not self.flow_level:571 572 # Are we allowed to start a key (not nessesary a simple)?573 if not self.allow_simple_key:574 raise ScannerError(575 None, None, 'mapping keys are not allowed here', self.reader.get_mark(),576 )577 578 # We may need to add BLOCK-MAPPING-START.579 if self.add_indent(self.reader.column):580 mark = self.reader.get_mark()581 self.tokens.append(tokens.BlockMappingStartToken(mark, mark))582 583 # Simple keys are allowed after '?' in the block context.584 self.allow_simple_key = not self.flow_level585 586 # Reset possible simple key on the current level.587 self.remove_possible_simple_key()588 589 # Add KEY.590 start_mark = self.reader.get_mark()591 self.reader.forward()592 end_mark = self.reader.get_mark()593 self.tokens.append(tokens.KeyToken(start_mark, end_mark))594 595 def fetch_value(self) -> None:596 # Do we determine a simple key?597 if self.flow_level in self.possible_simple_keys:598 # Add KEY.599 key = self.possible_simple_keys[self.flow_level]600 del self.possible_simple_keys[self.flow_level]601 self.tokens.insert(602 key.token_number - self.tokens_taken, tokens.KeyToken(key.mark, key.mark),603 )604 605 # If this key starts a new block mapping, we need to add606 # BLOCK-MAPPING-START.607 if not self.flow_level:608 if self.add_indent(key.column):609 self.tokens.insert(610 key.token_number - self.tokens_taken,611 tokens.BlockMappingStartToken(key.mark, key.mark),612 )613 614 # There cannot be two simple keys one after another.615 self.allow_simple_key = False616 617 # It must be a part of a complex key.618 else:619 620 # Block context needs additional checks.621 # (Do we really need them? They will be caught by the parser622 # anyway.)623 if not self.flow_level:624 625 # We are allowed to start a complex value if and only if626 # we can start a simple key.627 if not self.allow_simple_key:628 raise ScannerError(629 None,630 None,631 'mapping values are not allowed here',632 self.reader.get_mark(),633 )634 635 # If this value starts a new block mapping, we need to add636 # BLOCK-MAPPING-START. It will be detected as an error later by637 # the parser.638 if not self.flow_level:639 if self.add_indent(self.reader.column):640 mark = self.reader.get_mark()641 self.tokens.append(tokens.BlockMappingStartToken(mark, mark))642 643 # Simple keys are allowed after ':' in the block context.644 self.allow_simple_key = not self.flow_level645 646 # Reset possible simple key on the current level.647 self.remove_possible_simple_key()648 649 # Add VALUE.650 start_mark = self.reader.get_mark()651 self.reader.forward()652 end_mark = self.reader.get_mark()653 self.tokens.append(tokens.ValueToken(start_mark, end_mark))654 655 def fetch_alias(self) -> None:656 # ALIAS could be a simple key.657 self.save_possible_simple_key()658 # No simple keys after ALIAS.659 self.allow_simple_key = False660 # Scan and add ALIAS.661 self.tokens.append(self.scan_anchor(tokens.AliasToken))662 663 def fetch_anchor(self) -> None:664 # ANCHOR could start a simple key.665 self.save_possible_simple_key()666 # No simple keys after ANCHOR.667 self.allow_simple_key = False668 # Scan and add ANCHOR.669 self.tokens.append(self.scan_anchor(tokens.AnchorToken))670 671 def fetch_tag(self) -> None:672 # TAG could start a simple key.673 self.save_possible_simple_key()674 # No simple keys after TAG.675 self.allow_simple_key = False676 # Scan and add TAG.677 self.tokens.append(self.scan_tag())678 679 def fetch_literal(self) -> None:680 self.fetch_block_scalar(style='|')681 682 def fetch_folded(self) -> None:683 self.fetch_block_scalar(style='>')684 685 def fetch_block_scalar(self, style: Any) -> None:686 # A simple key may follow a block scalar.687 self.allow_simple_key = True688 # Reset possible simple key on the current level.689 self.remove_possible_simple_key()690 # Scan and add SCALAR.691 self.tokens.append(self.scan_block_scalar(style))692 693 def fetch_single(self) -> None:694 self.fetch_flow_scalar(style="'")695 696 def fetch_double(self) -> None:697 self.fetch_flow_scalar(style='"')698 699 def fetch_flow_scalar(self, style: Any) -> None:700 # A flow scalar could be a simple key.701 self.save_possible_simple_key()702 # No simple keys after flow scalars.703 self.allow_simple_key = False704 # Scan and add SCALAR.705 self.tokens.append(self.scan_flow_scalar(style))706 707 def fetch_plain(self) -> None:708 # A plain scalar could be a simple key.709 self.save_possible_simple_key()710 # No simple keys after plain scalars. But note that `scan_plain` will711 # change this flag if the scan is finished at the beginning of the712 # line.713 self.allow_simple_key = False714 # Scan and add SCALAR. May change `allow_simple_key`.715 self.tokens.append(self.scan_plain())716 717 # Checkers.718 719 def check_directive(self) -> Any:720 # DIRECTIVE: ^ '%' ...721 # The '%' indicator is already checked.722 if self.reader.column == 0:723 return True724 return None725 726 def check_document_start(self) -> Any:727 # DOCUMENT-START: ^ '---' (' '|'\n')728 if self.reader.column == 0:729 if self.reader.prefix(3) == '---' and self.reader.peek(3) in _THE_END_SPACE_TAB:730 return True731 return None732 733 def check_document_end(self) -> Any:734 # DOCUMENT-END: ^ '...' (' '|'\n')735 if self.reader.column == 0:736 if self.reader.prefix(3) == '...' and self.reader.peek(3) in _THE_END_SPACE_TAB:737 return True738 return None739 740 def check_block_entry(self) -> Any:741 # BLOCK-ENTRY: '-' (' '|'\n')742 return self.reader.peek(1) in _THE_END_SPACE_TAB743 744 def check_key(self) -> Any:745 # KEY(flow context): '?'746 if bool(self.flow_level):747 return True748 # KEY(block context): '?' (' '|'\n')749 return self.reader.peek(1) in _THE_END_SPACE_TAB750 751 def check_value(self) -> Any:752 # VALUE(flow context): ':'753 if self.scanner_processing_version == (1, 1):754 if bool(self.flow_level):755 return True756 else:757 if bool(self.flow_level):758 if self.flow_context[-1] == '[':759 if self.reader.peek(1) not in _THE_END_SPACE_TAB:760 return False761 elif self.tokens and isinstance(self.tokens[-1], tokens.ValueToken):762 # mapping flow context scanning a value token763 if self.reader.peek(1) not in _THE_END_SPACE_TAB:764 return False765 return True766 # VALUE(block context): ':' (' '|'\n')767 return self.reader.peek(1) in _THE_END_SPACE_TAB768 769 def check_plain(self) -> Any:770 # A plain scalar may start with any non-space character except:771 # '-', '?', ':', ',', '[', ']', '{', '}',772 # '#', '&', '*', '!', '|', '>', '\'', '\"',773 # '%', '@', '`'.774 #775 # It may also start with776 # '-', '?', ':'777 # if it is followed by a non-space character.778 #779 # Note that we limit the last rule to the block context (except the780 # '-' character) because we want the flow context to be space781 # independent.782 srp = self.reader.peek783 ch = srp()784 if self.scanner_processing_version == (1, 1):785 return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'"%@`' or (786 srp(1) not in _THE_END_SPACE_TAB787 and (ch == '-' or (not self.flow_level and ch in '?:'))788 )789 # YAML 1.2790 if ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'"%@`':791 # ################### ^ ???792 return True793 ch1 = srp(1)794 if ch == '-' and ch1 not in _THE_END_SPACE_TAB:795 return True796 if ch == ':' and bool(self.flow_level) and ch1 not in _SPACE_TAB:797 return True798 799 return srp(1) not in _THE_END_SPACE_TAB and (800 ch == '-' or (not self.flow_level and ch in '?:')801 )802 803 # Scanners.804 805 def scan_to_next_token(self) -> Any:806 # We ignore spaces, line breaks and comments.807 # If we find a line break in the block context, we set the flag808 # `allow_simple_key` on.809 # The byte order mark is stripped if it's the first character in the810 # stream. We do not yet support BOM inside the stream as the811 # specification requires. Any such mark will be considered as a part812 # of the document.813 #814 # TODO: We need to make tab handling rules more sane. A good rule is815 # Tabs cannot precede tokens816 # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,817 # KEY(block), VALUE(block), BLOCK-ENTRY818 # So the checking code is819 # if <TAB>:820 # self.allow_simple_keys = False821 # We also need to add the check for `allow_simple_keys == True` to822 # `unwind_indent` before issuing BLOCK-END.823 # Scanners for block, flow, and plain scalars need to be modified.824 srp = self.reader.peek825 srf = self.reader.forward826 if self.reader.index == 0 and srp() == '\uFEFF':827 srf()828 found = False829 _the_end = _THE_END830 white_space = ' \t' if self.flow_level > 0 else ' '831 while not found:832 while srp() in white_space:833 srf()834 if srp() == '#':835 while srp() not in _the_end:836 srf()837 if self.scan_line_break():838 if not self.flow_level:839 self.allow_simple_key = True840 else:841 found = True842 return None843 844 def scan_directive(self) -> Any:845 # See the specification for details.846 srp = self.reader.peek847 srf = self.reader.forward848 start_mark = self.reader.get_mark()849 srf()850 name = self.scan_directive_name(start_mark)851 value = None852 if name == 'YAML':853 value = self.scan_yaml_directive_value(start_mark)854 end_mark = self.reader.get_mark()855 elif name == 'TAG':856 value = self.scan_tag_directive_value(start_mark)857 end_mark = self.reader.get_mark()858 else:859 end_mark = self.reader.get_mark()860 while srp() not in _THE_END:861 srf()862 self.scan_directive_ignored_line(start_mark)863 return tokens.DirectiveToken(name, value, start_mark, end_mark)864 865 def scan_directive_name(self, start_mark: Any) -> Any:866 # See the specification for details.867 length = 0868 srp = self.reader.peek869 ch = srp(length)870 while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' or ch in '-_:.':871 length += 1872 ch = srp(length)873 if not length:874 raise ScannerError(875 'while scanning a directive',876 start_mark,877 f'expected alphabetic or numeric character, but found {ch!r}',878 self.reader.get_mark(),879 )880 value = self.reader.prefix(length)881 self.reader.forward(length)882 ch = srp()883 if ch not in '\0 \r\n\x85\u2028\u2029':884 raise ScannerError(885 'while scanning a directive',886 start_mark,887 f'expected alphabetic or numeric character, but found {ch!r}',888 self.reader.get_mark(),889 )890 return value891 892 def scan_yaml_directive_value(self, start_mark: Any) -> Any:893 # See the specification for details.894 srp = self.reader.peek895 srf = self.reader.forward896 while srp() == ' ':897 srf()898 major = self.scan_yaml_directive_number(start_mark)899 if srp() != '.':900 raise ScannerError(901 'while scanning a directive',902 start_mark,903 f"expected a digit or '.', but found {srp()!r}",904 self.reader.get_mark(),905 )906 srf()907 minor = self.scan_yaml_directive_number(start_mark)908 if srp() not in '\0 \r\n\x85\u2028\u2029':909 raise ScannerError(910 'while scanning a directive',911 start_mark,912 f"expected a digit or '.', but found {srp()!r}",913 self.reader.get_mark(),914 )915 self.yaml_version = (major, minor)916 self.loader.doc_infos[-1].doc_version = Version(major, minor)917 return self.yaml_version918 919 def scan_yaml_directive_number(self, start_mark: Any) -> Any:920 # See the specification for details.921 srp = self.reader.peek922 srf = self.reader.forward923 ch = srp()924 if not ('0' <= ch <= '9'):925 raise ScannerError(926 'while scanning a directive',927 start_mark,928 f'expected a digit, but found {ch!r}',929 self.reader.get_mark(),930 )931 length = 0932 while '0' <= srp(length) <= '9':933 length += 1934 value = int(self.reader.prefix(length))935 srf(length)936 return value937 938 def scan_tag_directive_value(self, start_mark: Any) -> Any:939 # See the specification for details.940 srp = self.reader.peek941 srf = self.reader.forward942 while srp() == ' ':943 srf()944 handle = self.scan_tag_directive_handle(start_mark)945 while srp() == ' ':946 srf()947 prefix = self.scan_tag_directive_prefix(start_mark)948 ret_val = (handle, prefix)949 self.tag_directives.append(ret_val)950 return ret_val951 952 def scan_tag_directive_handle(self, start_mark: Any) -> Any:953 # See the specification for details.954 value = self.scan_tag_handle('directive', start_mark)955 ch = self.reader.peek()956 if ch != ' ':957 raise ScannerError(958 'while scanning a directive',959 start_mark,960 f"expected ' ', but found {ch!r}",961 self.reader.get_mark(),962 )963 return value964 965 def scan_tag_directive_prefix(self, start_mark: Any) -> Any:966 # See the specification for details.967 value = self.scan_tag_uri('directive', start_mark)968 ch = self.reader.peek()969 if ch not in '\0 \r\n\x85\u2028\u2029':970 raise ScannerError(971 'while scanning a directive',972 start_mark,973 f"expected ' ', but found {ch!r}",974 self.reader.get_mark(),975 )976 return value977 978 def scan_directive_ignored_line(self, start_mark: Any) -> None:979 # See the specification for details.980 srp = self.reader.peek981 srf = self.reader.forward982 while srp() == ' ':983 srf()984 if srp() == '#':985 while srp() not in _THE_END:986 srf()987 ch = srp()988 if ch not in _THE_END:989 raise ScannerError(990 'while scanning a directive',991 start_mark,992 f'expected a comment or a line break, but found {ch!r}',993 self.reader.get_mark(),994 )995 self.scan_line_break()996 997 def scan_anchor(self, TokenClass: Any) -> Any:998 # The specification does not restrict characters for anchors and999 # aliases. This may lead to problems, for instance, the document:1000 # [ *alias, value ]1001 # can be interpteted in two ways, as1002 # [ "value" ]1003 # and1004 # [ *alias , "value" ]1005 # Therefore we restrict aliases to numbers and ASCII letters.1006 srp = self.reader.peek1007 start_mark = self.reader.get_mark()1008 indicator = srp()1009 if indicator == '*':1010 name = 'alias'1011 else:1012 name = 'anchor'1013 self.reader.forward()1014 length = 01015 ch = srp(length)1016 # while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \1017 # or ch in '-_':1018 while check_anchorname_char(ch):1019 length += 11020 ch = srp(length)1021 if not length:1022 raise ScannerError(1023 f'while scanning an {name!s}',1024 start_mark,1025 f'expected alphabetic or numeric character, but found {ch!r}',1026 self.reader.get_mark(),1027 )1028 value = self.reader.prefix(length)1029 self.reader.forward(length)1030 # ch1 = ch1031 # ch = srp() # no need to peek, ch is already set1032 # assert ch1 == ch1033 if ch not in '\0 \t\r\n\x85\u2028\u2029?:,[]{}%@`':1034 raise ScannerError(1035 f'while scanning an {name!s}',1036 start_mark,1037 f'expected alphabetic or numeric character, but found {ch!r}',1038 self.reader.get_mark(),1039 )1040 end_mark = self.reader.get_mark()1041 return TokenClass(value, start_mark, end_mark)1042 1043 def scan_tag(self) -> Any:1044 # See the specification for details.1045 srp = self.reader.peek1046 start_mark = self.reader.get_mark()1047 ch = srp(1)1048 short_handle = '!'1049 if ch == '!':1050 short_handle = '!!'1051 self.reader.forward()1052 srp = self.reader.peek1053 ch = srp(1)1054 1055 if ch == '<':1056 handle = None1057 self.reader.forward(2)1058 suffix = self.scan_tag_uri('tag', start_mark)1059 if srp() != '>':1060 raise ScannerError(1061 'while parsing a tag',1062 start_mark,1063 f"expected '>' but found {srp()!r}",1064 self.reader.get_mark(),1065 )1066 self.reader.forward()1067 elif ch in _THE_END_SPACE_TAB:1068 handle = None1069 suffix = short_handle1070 self.reader.forward()1071 else:1072 length = 11073 use_handle = False1074 while ch not in '\0 \r\n\x85\u2028\u2029':1075 if ch == '!':1076 use_handle = True1077 break1078 length += 11079 ch = srp(length)1080 handle = short_handle1081 if use_handle:1082 handle = self.scan_tag_handle('tag', start_mark)1083 else:1084 handle = short_handle1085 self.reader.forward()1086 suffix = self.scan_tag_uri('tag', start_mark)1087 ch = srp()1088 if ch not in '\0 \r\n\x85\u2028\u2029':1089 raise ScannerError(1090 'while scanning a tag',1091 start_mark,1092 f"expected ' ', but found {ch!r}",1093 self.reader.get_mark(),1094 )1095 value = (handle, suffix)1096 end_mark = self.reader.get_mark()1097 return tokens.TagToken(value, start_mark, end_mark)1098 1099 def scan_block_scalar(self, style: Any, rt: Optional[bool] = False) -> Any:1100 # See the specification for details.1101 srp = self.reader.peek1102 if style == '>':1103 folded = True1104 else:1105 folded = False1106 1107 chunks: List[Any] = []1108 start_mark = self.reader.get_mark()1109 1110 # Scan the header.1111 self.reader.forward()1112 chomping, increment = self.scan_block_scalar_indicators(start_mark)1113 # block scalar comment e.g. : |+ # comment text1114 block_scalar_comment = self.scan_block_scalar_ignored_line(start_mark)1115 1116 # Determine the indentation level and go to the first non-empty line.1117 min_indent = self.indent + 11118 if increment is None:1119 # no increment and top level, min_indent could be 01120 if min_indent < 1 and (1121 style not in '|>'1122 or (self.scanner_processing_version == (1, 1))1123 and getattr(1124 self.loader, 'top_level_block_style_scalar_no_indent_error_1_1', False,1125 )1126 ):1127 min_indent = 11128 breaks, max_indent, end_mark = self.scan_block_scalar_indentation()1129 indent = max(min_indent, max_indent)1130 else:1131 if min_indent < 1:1132 min_indent = 11133 indent = min_indent + increment - 11134 breaks, end_mark = self.scan_block_scalar_breaks(indent)1135 line_break = ""1136 1137 # Scan the inner part of the block scalar.1138 while self.reader.column == indent and srp() != '\0':1139 chunks.extend(breaks)1140 leading_non_space = srp() not in ' \t'1141 length = 01142 while srp(length) not in _THE_END:1143 length += 11144 chunks.append(self.reader.prefix(length))1145 self.reader.forward(length)1146 line_break = self.scan_line_break()1147 breaks, end_mark = self.scan_block_scalar_breaks(indent)1148 if style in '|>' and min_indent == 0:1149 # at the beginning of a line, if in block style see if1150 # end of document/start_new_document1151 if self.check_document_start() or self.check_document_end():1152 break1153 if self.reader.column == indent and srp() != '\0':1154 1155 # Unfortunately, folding rules are ambiguous.1156 #1157 # This is the folding according to the specification:1158 1159 if rt and folded and line_break == '\n':1160 chunks.append('\a')1161 if folded and line_break == '\n' and leading_non_space and srp() not in ' \t':1162 if not breaks:1163 chunks.append(' ')1164 else:1165 chunks.append(line_break)1166 1167 # This is Clark Evans's interpretation (also in the spec1168 # examples):1169 #1170 # if folded and line_break == '\n':1171 # if not breaks:1172 # if srp() not in ' \t':1173 # chunks.append(' ')1174 # else:1175 # chunks.append(line_break)1176 # else:1177 # chunks.append(line_break)1178 else:1179 break1180 1181 # Process trailing line breaks. The 'chomping' setting determines1182 # whether they are included in the value.1183 trailing: List[Any] = []1184 if chomping in [None, True]:1185 chunks.append(line_break)1186 if chomping is True:1187 chunks.extend(breaks)1188 elif chomping in [None, False]:1189 trailing.extend(breaks)1190 1191 # We are done.1192 token = tokens.ScalarToken("".join(chunks), False, start_mark, end_mark, style)1193 if self.loader is not None:1194 comment_handler = getattr(self.loader, 'comment_handling', False)1195 if comment_handler is None:1196 if block_scalar_comment is not None:1197 token.add_pre_comments([block_scalar_comment])1198 if len(trailing) > 0:1199 # Eat whitespaces and comments until we reach the next token.1200 if self.loader is not None: