codekingpro/portable-devtools
114k
1 2# Scanner produces tokens of the following types:3# STREAM-START4# STREAM-END5# DIRECTIVE(name, value)6# DOCUMENT-START7# DOCUMENT-END8# BLOCK-SEQUENCE-START9# BLOCK-MAPPING-START10# BLOCK-END11# FLOW-SEQUENCE-START12# FLOW-MAPPING-START13# FLOW-SEQUENCE-END14# FLOW-MAPPING-END15# BLOCK-ENTRY16# FLOW-ENTRY17# KEY18# VALUE19# ALIAS(value)20# ANCHOR(value)21# TAG(value)22# SCALAR(value, plain, style)23#24# Read comments in the Scanner code for more details.25#26 27__all__ = ['Scanner', 'ScannerError']28 29from .error import MarkedYAMLError30from .tokens import *31 32class ScannerError(MarkedYAMLError):33 pass34 35class SimpleKey:36 # See below simple keys treatment.37 38 def __init__(self, token_number, required, index, line, column, mark):39 self.token_number = token_number40 self.required = required41 self.index = index42 self.line = line43 self.column = column44 self.mark = mark45 46class Scanner:47 48 def __init__(self):49 """Initialize the scanner."""50 # It is assumed that Scanner and Reader will have a common descendant.51 # Reader do the dirty work of checking for BOM and converting the52 # input data to Unicode. It also adds NUL to the end.53 #54 # Reader supports the following methods55 # self.peek(i=0) # peek the next i-th character56 # self.prefix(l=1) # peek the next l characters57 # self.forward(l=1) # read the next l characters and move the pointer.58 59 # Had we reached the end of the stream?60 self.done = False61 62 # The number of unclosed '{' and '['. `flow_level == 0` means block63 # context.64 self.flow_level = 065 66 # List of processed tokens that are not yet emitted.67 self.tokens = []68 69 # Add the STREAM-START token.70 self.fetch_stream_start()71 72 # Number of tokens that were emitted through the `get_token` method.73 self.tokens_taken = 074 75 # The current indentation level.76 self.indent = -177 78 # Past indentation levels.79 self.indents = []80 81 # Variables related to simple keys treatment.82 83 # A simple key is a key that is not denoted by the '?' indicator.84 # Example of simple keys:85 # ---86 # block simple key: value87 # ? not a simple key:88 # : { flow simple key: value }89 # We emit the KEY token before all keys, so when we find a potential90 # simple key, we try to locate the corresponding ':' indicator.91 # Simple keys should be limited to a single line and 1024 characters.92 93 # Can a simple key start at the current position? A simple key may94 # start:95 # - at the beginning of the line, not counting indentation spaces96 # (in block context),97 # - after '{', '[', ',' (in the flow context),98 # - after '?', ':', '-' (in the block context).99 # In the block context, this flag also signifies if a block collection100 # may start at the current position.101 self.allow_simple_key = True102 103 # Keep track of possible simple keys. This is a dictionary. The key104 # is `flow_level`; there can be no more that one possible simple key105 # for each level. The value is a SimpleKey record:106 # (token_number, required, index, line, column, mark)107 # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),108 # '[', or '{' tokens.109 self.possible_simple_keys = {}110 111 # Public methods.112 113 def check_token(self, *choices):114 # Check if the next token is one of the given types.115 while self.need_more_tokens():116 self.fetch_more_tokens()117 if self.tokens:118 if not choices:119 return True120 for choice in choices:121 if isinstance(self.tokens[0], choice):122 return True123 return False124 125 def peek_token(self):126 # Return the next token, but do not delete if from the queue.127 # Return None if no more tokens.128 while self.need_more_tokens():129 self.fetch_more_tokens()130 if self.tokens:131 return self.tokens[0]132 else:133 return None134 135 def get_token(self):136 # Return the next token.137 while self.need_more_tokens():138 self.fetch_more_tokens()139 if self.tokens:140 self.tokens_taken += 1141 return self.tokens.pop(0)142 143 # Private methods.144 145 def need_more_tokens(self):146 if self.done:147 return False148 if not self.tokens:149 return True150 # The current token may be a potential simple key, so we151 # need to look further.152 self.stale_possible_simple_keys()153 if self.next_possible_simple_key() == self.tokens_taken:154 return True155 156 def fetch_more_tokens(self):157 158 # Eat whitespaces and comments until we reach the next token.159 self.scan_to_next_token()160 161 # Remove obsolete possible simple keys.162 self.stale_possible_simple_keys()163 164 # Compare the current indentation and column. It may add some tokens165 # and decrease the current indentation level.166 self.unwind_indent(self.column)167 168 # Peek the next character.169 ch = self.peek()170 171 # Is it the end of stream?172 if ch == '\0':173 return self.fetch_stream_end()174 175 # Is it a directive?176 if ch == '%' and self.check_directive():177 return self.fetch_directive()178 179 # Is it the document start?180 if ch == '-' and self.check_document_start():181 return self.fetch_document_start()182 183 # Is it the document end?184 if ch == '.' and self.check_document_end():185 return self.fetch_document_end()186 187 # TODO: support for BOM within a stream.188 #if ch == '\uFEFF':189 # return self.fetch_bom() <-- issue BOMToken190 191 # Note: the order of the following checks is NOT significant.192 193 # Is it the flow sequence start indicator?194 if ch == '[':195 return self.fetch_flow_sequence_start()196 197 # Is it the flow mapping start indicator?198 if ch == '{':199 return self.fetch_flow_mapping_start()200 201 # Is it the flow sequence end indicator?202 if ch == ']':203 return self.fetch_flow_sequence_end()204 205 # Is it the flow mapping end indicator?206 if ch == '}':207 return self.fetch_flow_mapping_end()208 209 # Is it the flow entry indicator?210 if ch == ',':211 return self.fetch_flow_entry()212 213 # Is it the block entry indicator?214 if ch == '-' and self.check_block_entry():215 return self.fetch_block_entry()216 217 # Is it the key indicator?218 if ch == '?' and self.check_key():219 return self.fetch_key()220 221 # Is it the value indicator?222 if ch == ':' and self.check_value():223 return self.fetch_value()224 225 # Is it an alias?226 if ch == '*':227 return self.fetch_alias()228 229 # Is it an anchor?230 if ch == '&':231 return self.fetch_anchor()232 233 # Is it a tag?234 if ch == '!':235 return self.fetch_tag()236 237 # Is it a literal scalar?238 if ch == '|' and not self.flow_level:239 return self.fetch_literal()240 241 # Is it a folded scalar?242 if ch == '>' and not self.flow_level:243 return self.fetch_folded()244 245 # Is it a single quoted scalar?246 if ch == '\'':247 return self.fetch_single()248 249 # Is it a double quoted scalar?250 if ch == '\"':251 return self.fetch_double()252 253 # It must be a plain scalar then.254 if self.check_plain():255 return self.fetch_plain()256 257 # No? It's an error. Let's produce a nice error message.258 raise ScannerError("while scanning for the next token", None,259 "found character %r that cannot start any token" % ch,260 self.get_mark())261 262 # Simple keys treatment.263 264 def next_possible_simple_key(self):265 # Return the number of the nearest possible simple key. Actually we266 # don't need to loop through the whole dictionary. We may replace it267 # with the following code:268 # if not self.possible_simple_keys:269 # return None270 # return self.possible_simple_keys[271 # min(self.possible_simple_keys.keys())].token_number272 min_token_number = None273 for level in self.possible_simple_keys:274 key = self.possible_simple_keys[level]275 if min_token_number is None or key.token_number < min_token_number:276 min_token_number = key.token_number277 return min_token_number278 279 def stale_possible_simple_keys(self):280 # Remove entries that are no longer possible simple keys. According to281 # the YAML specification, simple keys282 # - should be limited to a single line,283 # - should be no longer than 1024 characters.284 # Disabling this procedure will allow simple keys of any length and285 # height (may cause problems if indentation is broken though).286 for level in list(self.possible_simple_keys):287 key = self.possible_simple_keys[level]288 if key.line != self.line \289 or self.index-key.index > 1024:290 if key.required:291 raise ScannerError("while scanning a simple key", key.mark,292 "could not find expected ':'", self.get_mark())293 del self.possible_simple_keys[level]294 295 def save_possible_simple_key(self):296 # The next token may start a simple key. We check if it's possible297 # and save its position. This function is called for298 # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.299 300 # Check if a simple key is required at the current position.301 required = not self.flow_level and self.indent == self.column302 303 # The next token might be a simple key. Let's save it's number and304 # position.305 if self.allow_simple_key:306 self.remove_possible_simple_key()307 token_number = self.tokens_taken+len(self.tokens)308 key = SimpleKey(token_number, required,309 self.index, self.line, self.column, self.get_mark())310 self.possible_simple_keys[self.flow_level] = key311 312 def remove_possible_simple_key(self):313 # Remove the saved possible key position at the current flow level.314 if self.flow_level in self.possible_simple_keys:315 key = self.possible_simple_keys[self.flow_level]316 317 if key.required:318 raise ScannerError("while scanning a simple key", key.mark,319 "could not find expected ':'", self.get_mark())320 321 del self.possible_simple_keys[self.flow_level]322 323 # Indentation functions.324 325 def unwind_indent(self, column):326 327 ## In flow context, tokens should respect indentation.328 ## Actually the condition should be `self.indent >= column` according to329 ## the spec. But this condition will prohibit intuitively correct330 ## constructions such as331 ## key : {332 ## }333 #if self.flow_level and self.indent > column:334 # raise ScannerError(None, None,335 # "invalid indentation or unclosed '[' or '{'",336 # self.get_mark())337 338 # In the flow context, indentation is ignored. We make the scanner less339 # restrictive then specification requires.340 if self.flow_level:341 return342 343 # In block context, we may need to issue the BLOCK-END tokens.344 while self.indent > column:345 mark = self.get_mark()346 self.indent = self.indents.pop()347 self.tokens.append(BlockEndToken(mark, mark))348 349 def add_indent(self, column):350 # Check if we need to increase indentation.351 if self.indent < column:352 self.indents.append(self.indent)353 self.indent = column354 return True355 return False356 357 # Fetchers.358 359 def fetch_stream_start(self):360 # We always add STREAM-START as the first token and STREAM-END as the361 # last token.362 363 # Read the token.364 mark = self.get_mark()365 366 # Add STREAM-START.367 self.tokens.append(StreamStartToken(mark, mark,368 encoding=self.encoding))369 370 371 def fetch_stream_end(self):372 373 # Set the current indentation to -1.374 self.unwind_indent(-1)375 376 # Reset simple keys.377 self.remove_possible_simple_key()378 self.allow_simple_key = False379 self.possible_simple_keys = {}380 381 # Read the token.382 mark = self.get_mark()383 384 # Add STREAM-END.385 self.tokens.append(StreamEndToken(mark, mark))386 387 # The steam is finished.388 self.done = True389 390 def fetch_directive(self):391 392 # Set the current indentation to -1.393 self.unwind_indent(-1)394 395 # Reset simple keys.396 self.remove_possible_simple_key()397 self.allow_simple_key = False398 399 # Scan and add DIRECTIVE.400 self.tokens.append(self.scan_directive())401 402 def fetch_document_start(self):403 self.fetch_document_indicator(DocumentStartToken)404 405 def fetch_document_end(self):406 self.fetch_document_indicator(DocumentEndToken)407 408 def fetch_document_indicator(self, TokenClass):409 410 # Set the current indentation to -1.411 self.unwind_indent(-1)412 413 # Reset simple keys. Note that there could not be a block collection414 # after '---'.415 self.remove_possible_simple_key()416 self.allow_simple_key = False417 418 # Add DOCUMENT-START or DOCUMENT-END.419 start_mark = self.get_mark()420 self.forward(3)421 end_mark = self.get_mark()422 self.tokens.append(TokenClass(start_mark, end_mark))423 424 def fetch_flow_sequence_start(self):425 self.fetch_flow_collection_start(FlowSequenceStartToken)426 427 def fetch_flow_mapping_start(self):428 self.fetch_flow_collection_start(FlowMappingStartToken)429 430 def fetch_flow_collection_start(self, TokenClass):431 432 # '[' and '{' may start a simple key.433 self.save_possible_simple_key()434 435 # Increase the flow level.436 self.flow_level += 1437 438 # Simple keys are allowed after '[' and '{'.439 self.allow_simple_key = True440 441 # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.442 start_mark = self.get_mark()443 self.forward()444 end_mark = self.get_mark()445 self.tokens.append(TokenClass(start_mark, end_mark))446 447 def fetch_flow_sequence_end(self):448 self.fetch_flow_collection_end(FlowSequenceEndToken)449 450 def fetch_flow_mapping_end(self):451 self.fetch_flow_collection_end(FlowMappingEndToken)452 453 def fetch_flow_collection_end(self, TokenClass):454 455 # Reset possible simple key on the current level.456 self.remove_possible_simple_key()457 458 # Decrease the flow level.459 self.flow_level -= 1460 461 # No simple keys after ']' or '}'.462 self.allow_simple_key = False463 464 # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.465 start_mark = self.get_mark()466 self.forward()467 end_mark = self.get_mark()468 self.tokens.append(TokenClass(start_mark, end_mark))469 470 def fetch_flow_entry(self):471 472 # Simple keys are allowed after ','.473 self.allow_simple_key = True474 475 # Reset possible simple key on the current level.476 self.remove_possible_simple_key()477 478 # Add FLOW-ENTRY.479 start_mark = self.get_mark()480 self.forward()481 end_mark = self.get_mark()482 self.tokens.append(FlowEntryToken(start_mark, end_mark))483 484 def fetch_block_entry(self):485 486 # Block context needs additional checks.487 if not self.flow_level:488 489 # Are we allowed to start a new entry?490 if not self.allow_simple_key:491 raise ScannerError(None, None,492 "sequence entries are not allowed here",493 self.get_mark())494 495 # We may need to add BLOCK-SEQUENCE-START.496 if self.add_indent(self.column):497 mark = self.get_mark()498 self.tokens.append(BlockSequenceStartToken(mark, mark))499 500 # It's an error for the block entry to occur in the flow context,501 # but we let the parser detect this.502 else:503 pass504 505 # Simple keys are allowed after '-'.506 self.allow_simple_key = True507 508 # Reset possible simple key on the current level.509 self.remove_possible_simple_key()510 511 # Add BLOCK-ENTRY.512 start_mark = self.get_mark()513 self.forward()514 end_mark = self.get_mark()515 self.tokens.append(BlockEntryToken(start_mark, end_mark))516 517 def fetch_key(self):518 519 # Block context needs additional checks.520 if not self.flow_level:521 522 # Are we allowed to start a key (not necessary a simple)?523 if not self.allow_simple_key:524 raise ScannerError(None, None,525 "mapping keys are not allowed here",526 self.get_mark())527 528 # We may need to add BLOCK-MAPPING-START.529 if self.add_indent(self.column):530 mark = self.get_mark()531 self.tokens.append(BlockMappingStartToken(mark, mark))532 533 # Simple keys are allowed after '?' in the block context.534 self.allow_simple_key = not self.flow_level535 536 # Reset possible simple key on the current level.537 self.remove_possible_simple_key()538 539 # Add KEY.540 start_mark = self.get_mark()541 self.forward()542 end_mark = self.get_mark()543 self.tokens.append(KeyToken(start_mark, end_mark))544 545 def fetch_value(self):546 547 # Do we determine a simple key?548 if self.flow_level in self.possible_simple_keys:549 550 # Add KEY.551 key = self.possible_simple_keys[self.flow_level]552 del self.possible_simple_keys[self.flow_level]553 self.tokens.insert(key.token_number-self.tokens_taken,554 KeyToken(key.mark, key.mark))555 556 # If this key starts a new block mapping, we need to add557 # BLOCK-MAPPING-START.558 if not self.flow_level:559 if self.add_indent(key.column):560 self.tokens.insert(key.token_number-self.tokens_taken,561 BlockMappingStartToken(key.mark, key.mark))562 563 # There cannot be two simple keys one after another.564 self.allow_simple_key = False565 566 # It must be a part of a complex key.567 else:568 569 # Block context needs additional checks.570 # (Do we really need them? They will be caught by the parser571 # anyway.)572 if not self.flow_level:573 574 # We are allowed to start a complex value if and only if575 # we can start a simple key.576 if not self.allow_simple_key:577 raise ScannerError(None, None,578 "mapping values are not allowed here",579 self.get_mark())580 581 # If this value starts a new block mapping, we need to add582 # BLOCK-MAPPING-START. It will be detected as an error later by583 # the parser.584 if not self.flow_level:585 if self.add_indent(self.column):586 mark = self.get_mark()587 self.tokens.append(BlockMappingStartToken(mark, mark))588 589 # Simple keys are allowed after ':' in the block context.590 self.allow_simple_key = not self.flow_level591 592 # Reset possible simple key on the current level.593 self.remove_possible_simple_key()594 595 # Add VALUE.596 start_mark = self.get_mark()597 self.forward()598 end_mark = self.get_mark()599 self.tokens.append(ValueToken(start_mark, end_mark))600 601 def fetch_alias(self):602 603 # ALIAS could be a simple key.604 self.save_possible_simple_key()605 606 # No simple keys after ALIAS.607 self.allow_simple_key = False608 609 # Scan and add ALIAS.610 self.tokens.append(self.scan_anchor(AliasToken))611 612 def fetch_anchor(self):613 614 # ANCHOR could start a simple key.615 self.save_possible_simple_key()616 617 # No simple keys after ANCHOR.618 self.allow_simple_key = False619 620 # Scan and add ANCHOR.621 self.tokens.append(self.scan_anchor(AnchorToken))622 623 def fetch_tag(self):624 625 # TAG could start a simple key.626 self.save_possible_simple_key()627 628 # No simple keys after TAG.629 self.allow_simple_key = False630 631 # Scan and add TAG.632 self.tokens.append(self.scan_tag())633 634 def fetch_literal(self):635 self.fetch_block_scalar(style='|')636 637 def fetch_folded(self):638 self.fetch_block_scalar(style='>')639 640 def fetch_block_scalar(self, style):641 642 # A simple key may follow a block scalar.643 self.allow_simple_key = True644 645 # Reset possible simple key on the current level.646 self.remove_possible_simple_key()647 648 # Scan and add SCALAR.649 self.tokens.append(self.scan_block_scalar(style))650 651 def fetch_single(self):652 self.fetch_flow_scalar(style='\'')653 654 def fetch_double(self):655 self.fetch_flow_scalar(style='"')656 657 def fetch_flow_scalar(self, style):658 659 # A flow scalar could be a simple key.660 self.save_possible_simple_key()661 662 # No simple keys after flow scalars.663 self.allow_simple_key = False664 665 # Scan and add SCALAR.666 self.tokens.append(self.scan_flow_scalar(style))667 668 def fetch_plain(self):669 670 # A plain scalar could be a simple key.671 self.save_possible_simple_key()672 673 # No simple keys after plain scalars. But note that `scan_plain` will674 # change this flag if the scan is finished at the beginning of the675 # line.676 self.allow_simple_key = False677 678 # Scan and add SCALAR. May change `allow_simple_key`.679 self.tokens.append(self.scan_plain())680 681 # Checkers.682 683 def check_directive(self):684 685 # DIRECTIVE: ^ '%' ...686 # The '%' indicator is already checked.687 if self.column == 0:688 return True689 690 def check_document_start(self):691 692 # DOCUMENT-START: ^ '---' (' '|'\n')693 if self.column == 0:694 if self.prefix(3) == '---' \695 and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':696 return True697 698 def check_document_end(self):699 700 # DOCUMENT-END: ^ '...' (' '|'\n')701 if self.column == 0:702 if self.prefix(3) == '...' \703 and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':704 return True705 706 def check_block_entry(self):707 708 # BLOCK-ENTRY: '-' (' '|'\n')709 return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'710 711 def check_key(self):712 713 # KEY(flow context): '?'714 if self.flow_level:715 return True716 717 # KEY(block context): '?' (' '|'\n')718 else:719 return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'720 721 def check_value(self):722 723 # VALUE(flow context): ':'724 if self.flow_level:725 return True726 727 # VALUE(block context): ':' (' '|'\n')728 else:729 return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'730 731 def check_plain(self):732 733 # A plain scalar may start with any non-space character except:734 # '-', '?', ':', ',', '[', ']', '{', '}',735 # '#', '&', '*', '!', '|', '>', '\'', '\"',736 # '%', '@', '`'.737 #738 # It may also start with739 # '-', '?', ':'740 # if it is followed by a non-space character.741 #742 # Note that we limit the last rule to the block context (except the743 # '-' character) because we want the flow context to be space744 # independent.745 ch = self.peek()746 return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \747 or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029'748 and (ch == '-' or (not self.flow_level and ch in '?:')))749 750 # Scanners.751 752 def scan_to_next_token(self):753 # We ignore spaces, line breaks and comments.754 # If we find a line break in the block context, we set the flag755 # `allow_simple_key` on.756 # The byte order mark is stripped if it's the first character in the757 # stream. We do not yet support BOM inside the stream as the758 # specification requires. Any such mark will be considered as a part759 # of the document.760 #761 # TODO: We need to make tab handling rules more sane. A good rule is762 # Tabs cannot precede tokens763 # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,764 # KEY(block), VALUE(block), BLOCK-ENTRY765 # So the checking code is766 # if <TAB>:767 # self.allow_simple_keys = False768 # We also need to add the check for `allow_simple_keys == True` to769 # `unwind_indent` before issuing BLOCK-END.770 # Scanners for block, flow, and plain scalars need to be modified.771 772 if self.index == 0 and self.peek() == '\uFEFF':773 self.forward()774 found = False775 while not found:776 while self.peek() == ' ':777 self.forward()778 if self.peek() == '#':779 while self.peek() not in '\0\r\n\x85\u2028\u2029':780 self.forward()781 if self.scan_line_break():782 if not self.flow_level:783 self.allow_simple_key = True784 else:785 found = True786 787 def scan_directive(self):788 # See the specification for details.789 start_mark = self.get_mark()790 self.forward()791 name = self.scan_directive_name(start_mark)792 value = None793 if name == 'YAML':794 value = self.scan_yaml_directive_value(start_mark)795 end_mark = self.get_mark()796 elif name == 'TAG':797 value = self.scan_tag_directive_value(start_mark)798 end_mark = self.get_mark()799 else:800 end_mark = self.get_mark()801 while self.peek() not in '\0\r\n\x85\u2028\u2029':802 self.forward()803 self.scan_directive_ignored_line(start_mark)804 return DirectiveToken(name, value, start_mark, end_mark)805 806 def scan_directive_name(self, start_mark):807 # See the specification for details.808 length = 0809 ch = self.peek(length)810 while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \811 or ch in '-_':812 length += 1813 ch = self.peek(length)814 if not length:815 raise ScannerError("while scanning a directive", start_mark,816 "expected alphabetic or numeric character, but found %r"817 % ch, self.get_mark())818 value = self.prefix(length)819 self.forward(length)820 ch = self.peek()821 if ch not in '\0 \r\n\x85\u2028\u2029':822 raise ScannerError("while scanning a directive", start_mark,823 "expected alphabetic or numeric character, but found %r"824 % ch, self.get_mark())825 return value826 827 def scan_yaml_directive_value(self, start_mark):828 # See the specification for details.829 while self.peek() == ' ':830 self.forward()831 major = self.scan_yaml_directive_number(start_mark)832 if self.peek() != '.':833 raise ScannerError("while scanning a directive", start_mark,834 "expected a digit or '.', but found %r" % self.peek(),835 self.get_mark())836 self.forward()837 minor = self.scan_yaml_directive_number(start_mark)838 if self.peek() not in '\0 \r\n\x85\u2028\u2029':839 raise ScannerError("while scanning a directive", start_mark,840 "expected a digit or ' ', but found %r" % self.peek(),841 self.get_mark())842 return (major, minor)843 844 def scan_yaml_directive_number(self, start_mark):845 # See the specification for details.846 ch = self.peek()847 if not ('0' <= ch <= '9'):848 raise ScannerError("while scanning a directive", start_mark,849 "expected a digit, but found %r" % ch, self.get_mark())850 length = 0851 while '0' <= self.peek(length) <= '9':852 length += 1853 value = int(self.prefix(length))854 self.forward(length)855 return value856 857 def scan_tag_directive_value(self, start_mark):858 # See the specification for details.859 while self.peek() == ' ':860 self.forward()861 handle = self.scan_tag_directive_handle(start_mark)862 while self.peek() == ' ':863 self.forward()864 prefix = self.scan_tag_directive_prefix(start_mark)865 return (handle, prefix)866 867 def scan_tag_directive_handle(self, start_mark):868 # See the specification for details.869 value = self.scan_tag_handle('directive', start_mark)870 ch = self.peek()871 if ch != ' ':872 raise ScannerError("while scanning a directive", start_mark,873 "expected ' ', but found %r" % ch, self.get_mark())874 return value875 876 def scan_tag_directive_prefix(self, start_mark):877 # See the specification for details.878 value = self.scan_tag_uri('directive', start_mark)879 ch = self.peek()880 if ch not in '\0 \r\n\x85\u2028\u2029':881 raise ScannerError("while scanning a directive", start_mark,882 "expected ' ', but found %r" % ch, self.get_mark())883 return value884 885 def scan_directive_ignored_line(self, start_mark):886 # See the specification for details.887 while self.peek() == ' ':888 self.forward()889 if self.peek() == '#':890 while self.peek() not in '\0\r\n\x85\u2028\u2029':891 self.forward()892 ch = self.peek()893 if ch not in '\0\r\n\x85\u2028\u2029':894 raise ScannerError("while scanning a directive", start_mark,895 "expected a comment or a line break, but found %r"896 % ch, self.get_mark())897 self.scan_line_break()898 899 def scan_anchor(self, TokenClass):900 # The specification does not restrict characters for anchors and901 # aliases. This may lead to problems, for instance, the document:902 # [ *alias, value ]903 # can be interpreted in two ways, as904 # [ "value" ]905 # and906 # [ *alias , "value" ]907 # Therefore we restrict aliases to numbers and ASCII letters.908 start_mark = self.get_mark()909 indicator = self.peek()910 if indicator == '*':911 name = 'alias'912 else:913 name = 'anchor'914 self.forward()915 length = 0916 ch = self.peek(length)917 while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \918 or ch in '-_':919 length += 1920 ch = self.peek(length)921 if not length:922 raise ScannerError("while scanning an %s" % name, start_mark,923 "expected alphabetic or numeric character, but found %r"924 % ch, self.get_mark())925 value = self.prefix(length)926 self.forward(length)927 ch = self.peek()928 if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`':929 raise ScannerError("while scanning an %s" % name, start_mark,930 "expected alphabetic or numeric character, but found %r"931 % ch, self.get_mark())932 end_mark = self.get_mark()933 return TokenClass(value, start_mark, end_mark)934 935 def scan_tag(self):936 # See the specification for details.937 start_mark = self.get_mark()938 ch = self.peek(1)939 if ch == '<':940 handle = None941 self.forward(2)942 suffix = self.scan_tag_uri('tag', start_mark)943 if self.peek() != '>':944 raise ScannerError("while parsing a tag", start_mark,945 "expected '>', but found %r" % self.peek(),946 self.get_mark())947 self.forward()948 elif ch in '\0 \t\r\n\x85\u2028\u2029':949 handle = None950 suffix = '!'951 self.forward()952 else:953 length = 1954 use_handle = False955 while ch not in '\0 \r\n\x85\u2028\u2029':956 if ch == '!':957 use_handle = True958 break959 length += 1960 ch = self.peek(length)961 handle = '!'962 if use_handle:963 handle = self.scan_tag_handle('tag', start_mark)964 else:965 handle = '!'966 self.forward()967 suffix = self.scan_tag_uri('tag', start_mark)968 ch = self.peek()969 if ch not in '\0 \r\n\x85\u2028\u2029':970 raise ScannerError("while scanning a tag", start_mark,971 "expected ' ', but found %r" % ch, self.get_mark())972 value = (handle, suffix)973 end_mark = self.get_mark()974 return TagToken(value, start_mark, end_mark)975 976 def scan_block_scalar(self, style):977 # See the specification for details.978 979 if style == '>':980 folded = True981 else:982 folded = False983 984 chunks = []985 start_mark = self.get_mark()986 987 # Scan the header.988 self.forward()989 chomping, increment = self.scan_block_scalar_indicators(start_mark)990 self.scan_block_scalar_ignored_line(start_mark)991 992 # Determine the indentation level and go to the first non-empty line.993 min_indent = self.indent+1994 if min_indent < 1:995 min_indent = 1996 if increment is None:997 breaks, max_indent, end_mark = self.scan_block_scalar_indentation()998 indent = max(min_indent, max_indent)999 else:1000 indent = min_indent+increment-11001 breaks, end_mark = self.scan_block_scalar_breaks(indent)1002 line_break = ''1003 1004 # Scan the inner part of the block scalar.1005 while self.column == indent and self.peek() != '\0':1006 chunks.extend(breaks)1007 leading_non_space = self.peek() not in ' \t'1008 length = 01009 while self.peek(length) not in '\0\r\n\x85\u2028\u2029':1010 length += 11011 chunks.append(self.prefix(length))1012 self.forward(length)1013 line_break = self.scan_line_break()1014 breaks, end_mark = self.scan_block_scalar_breaks(indent)1015 if self.column == indent and self.peek() != '\0':1016 1017 # Unfortunately, folding rules are ambiguous.1018 #1019 # This is the folding according to the specification:1020 1021 if folded and line_break == '\n' \1022 and leading_non_space and self.peek() not in ' \t':1023 if not breaks:1024 chunks.append(' ')1025 else:1026 chunks.append(line_break)1027 1028 # This is Clark Evans's interpretation (also in the spec1029 # examples):1030 #1031 #if folded and line_break == '\n':1032 # if not breaks:1033 # if self.peek() not in ' \t':1034 # chunks.append(' ')1035 # else:1036 # chunks.append(line_break)1037 #else:1038 # chunks.append(line_break)1039 else:1040 break1041 1042 # Chomp the tail.1043 if chomping is not False:1044 chunks.append(line_break)1045 if chomping is True:1046 chunks.extend(breaks)1047 1048 # We are done.1049 return ScalarToken(''.join(chunks), False, start_mark, end_mark,1050 style)1051 1052 def scan_block_scalar_indicators(self, start_mark):1053 # See the specification for details.1054 chomping = None1055 increment = None1056 ch = self.peek()1057 if ch in '+-':1058 if ch == '+':1059 chomping = True1060 else:1061 chomping = False1062 self.forward()1063 ch = self.peek()1064 if ch in '0123456789':1065 increment = int(ch)1066 if increment == 0:1067 raise ScannerError("while scanning a block scalar", start_mark,1068 "expected indentation indicator in the range 1-9, but found 0",1069 self.get_mark())1070 self.forward()1071 elif ch in '0123456789':1072 increment = int(ch)1073 if increment == 0:1074 raise ScannerError("while scanning a block scalar", start_mark,1075 "expected indentation indicator in the range 1-9, but found 0",1076 self.get_mark())1077 self.forward()1078 ch = self.peek()1079 if ch in '+-':1080 if ch == '+':1081 chomping = True1082 else:1083 chomping = False1084 self.forward()1085 ch = self.peek()1086 if ch not in '\0 \r\n\x85\u2028\u2029':1087 raise ScannerError("while scanning a block scalar", start_mark,1088 "expected chomping or indentation indicators, but found %r"1089 % ch, self.get_mark())1090 return chomping, increment1091 1092 def scan_block_scalar_ignored_line(self, start_mark):1093 # See the specification for details.1094 while self.peek() == ' ':1095 self.forward()1096 if self.peek() == '#':1097 while self.peek() not in '\0\r\n\x85\u2028\u2029':1098 self.forward()1099 ch = self.peek()1100 if ch not in '\0\r\n\x85\u2028\u2029':1101 raise ScannerError("while scanning a block scalar", start_mark,1102 "expected a comment or a line break, but found %r" % ch,1103 self.get_mark())1104 self.scan_line_break()1105 1106 def scan_block_scalar_indentation(self):1107 # See the specification for details.1108 chunks = []1109 max_indent = 01110 end_mark = self.get_mark()1111 while self.peek() in ' \r\n\x85\u2028\u2029':1112 if self.peek() != ' ':1113 chunks.append(self.scan_line_break())1114 end_mark = self.get_mark()1115 else:1116 self.forward()1117 if self.column > max_indent:1118 max_indent = self.column1119 return chunks, max_indent, end_mark1120 1121 def scan_block_scalar_breaks(self, indent):1122 # See the specification for details.1123 chunks = []1124 end_mark = self.get_mark()1125 while self.column < indent and self.peek() == ' ':1126 self.forward()1127 while self.peek() in '\r\n\x85\u2028\u2029':1128 chunks.append(self.scan_line_break())1129 end_mark = self.get_mark()1130 while self.column < indent and self.peek() == ' ':1131 self.forward()1132 return chunks, end_mark1133 1134 def scan_flow_scalar(self, style):1135 # See the specification for details.1136 # Note that we loose indentation rules for quoted scalars. Quoted1137 # scalars don't need to adhere indentation because " and ' clearly1138 # mark the beginning and the end of them. Therefore we are less1139 # restrictive then the specification requires. We only need to check1140 # that document separators are not included in scalars.1141 if style == '"':1142 double = True1143 else:1144 double = False1145 chunks = []1146 start_mark = self.get_mark()1147 quote = self.peek()1148 self.forward()1149 chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))1150 while self.peek() != quote:1151 chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))1152 chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))1153 self.forward()1154 end_mark = self.get_mark()1155 return ScalarToken(''.join(chunks), False, start_mark, end_mark,1156 style)1157 1158 ESCAPE_REPLACEMENTS = {1159 '0': '\0',1160 'a': '\x07',1161 'b': '\x08',1162 't': '\x09',1163 '\t': '\x09',1164 'n': '\x0A',1165 'v': '\x0B',1166 'f': '\x0C',1167 'r': '\x0D',1168 'e': '\x1B',1169 ' ': '\x20',1170 '\"': '\"',1171 '\\': '\\',1172 '/': '/',1173 'N': '\x85',1174 '_': '\xA0',1175 'L': '\u2028',1176 'P': '\u2029',1177 }1178 1179 ESCAPE_CODES = {1180 'x': 2,1181 'u': 4,1182 'U': 8,1183 }1184 1185 def scan_flow_scalar_non_spaces(self, double, start_mark):1186 # See the specification for details.1187 chunks = []1188 while True:1189 length = 01190 while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':1191 length += 11192 if length:1193 chunks.append(self.prefix(length))1194 self.forward(length)1195 ch = self.peek()1196 if not double and ch == '\'' and self.peek(1) == '\'':1197 chunks.append('\'')1198 self.forward(2)1199 elif (double and ch == '\'') or (not double and ch in '\"\\'):1200 chunks.append(ch)