codekingpro/portable-devtools
114k
1# Protocol Buffers - Google's data interchange format2# Copyright 2008 Google Inc. All rights reserved.3#4# Use of this source code is governed by a BSD-style5# license that can be found in the LICENSE file or at6# https://developers.google.com/open-source/licenses/bsd7 8"""Contains routines for printing protocol messages in text format.9 10Simple usage example::11 12 # Create a proto object and serialize it to a text proto string.13 message = my_proto_pb2.MyMessage(foo='bar')14 text_proto = text_format.MessageToString(message)15 16 # Parse a text proto string.17 message = text_format.Parse(text_proto, my_proto_pb2.MyMessage())18"""19 20__author__ = 'kenton@google.com (Kenton Varda)'21 22# TODO Import thread contention leads to test failures.23import encodings.raw_unicode_escape # pylint: disable=unused-import24import encodings.unicode_escape # pylint: disable=unused-import25import io26import math27import re28 29from google.protobuf.internal import decoder30from google.protobuf.internal import type_checkers31from google.protobuf import descriptor32from google.protobuf import text_encoding33from google.protobuf import unknown_fields34 35# pylint: disable=g-import-not-at-top36__all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField',37 'PrintFieldValue', 'Merge', 'MessageToBytes']38 39_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),40 type_checkers.Int32ValueChecker(),41 type_checkers.Uint64ValueChecker(),42 type_checkers.Int64ValueChecker())43_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?$', re.IGNORECASE)44_FLOAT_NAN = re.compile('nanf?$', re.IGNORECASE)45_QUOTES = frozenset(("'", '"'))46_ANY_FULL_TYPE_NAME = 'google.protobuf.Any'47_DEBUG_STRING_SILENT_MARKER = '\t '48 49 50class Error(Exception):51 """Top-level module error for text_format."""52 53 54class ParseError(Error):55 """Thrown in case of text parsing or tokenizing error."""56 57 def __init__(self, message=None, line=None, column=None):58 if message is not None and line is not None:59 loc = str(line)60 if column is not None:61 loc += ':{0}'.format(column)62 message = '{0} : {1}'.format(loc, message)63 if message is not None:64 super(ParseError, self).__init__(message)65 else:66 super(ParseError, self).__init__()67 self._line = line68 self._column = column69 70 def GetLine(self):71 return self._line72 73 def GetColumn(self):74 return self._column75 76 77class TextWriter(object):78 79 def __init__(self, as_utf8):80 self._writer = io.StringIO()81 82 def write(self, val):83 return self._writer.write(val)84 85 def close(self):86 return self._writer.close()87 88 def getvalue(self):89 return self._writer.getvalue()90 91 92def MessageToString(93 message,94 as_utf8=False,95 as_one_line=False,96 use_short_repeated_primitives=False,97 pointy_brackets=False,98 use_index_order=False,99 float_format=None,100 double_format=None,101 use_field_number=False,102 descriptor_pool=None,103 indent=0,104 message_formatter=None,105 print_unknown_fields=False,106 force_colon=False) -> str:107 """Convert protobuf message to text format.108 109 Double values can be formatted compactly with 15 digits of110 precision (which is the most that IEEE 754 "double" can guarantee)111 using double_format='.15g'. To ensure that converting to text and back to a112 proto will result in an identical value, double_format='.17g' should be used.113 114 Args:115 message: The protocol buffers message.116 as_utf8: Return unescaped Unicode for non-ASCII characters.117 as_one_line: Don't introduce newlines between fields.118 use_short_repeated_primitives: Use short repeated format for primitives.119 pointy_brackets: If True, use angle brackets instead of curly braces for120 nesting.121 use_index_order: If True, fields of a proto message will be printed using122 the order defined in source code instead of the field number, extensions123 will be printed at the end of the message and their relative order is124 determined by the extension number. By default, use the field number125 order.126 float_format (str): If set, use this to specify float field formatting127 (per the "Format Specification Mini-Language"); otherwise, shortest float128 that has same value in wire will be printed. Also affect double field129 if double_format is not set but float_format is set.130 double_format (str): If set, use this to specify double field formatting131 (per the "Format Specification Mini-Language"); if it is not set but132 float_format is set, use float_format. Otherwise, use ``str()``133 use_field_number: If True, print field numbers instead of names.134 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.135 indent (int): The initial indent level, in terms of spaces, for pretty136 print.137 message_formatter (function(message, indent, as_one_line) -> unicode|None):138 Custom formatter for selected sub-messages (usually based on message139 type). Use to pretty print parts of the protobuf for easier diffing.140 print_unknown_fields: If True, unknown fields will be printed.141 force_colon: If set, a colon will be added after the field name even if the142 field is a proto message.143 144 Returns:145 str: A string of the text formatted protocol buffer message.146 """147 out = TextWriter(as_utf8)148 printer = _Printer(149 out,150 indent,151 as_utf8,152 as_one_line,153 use_short_repeated_primitives,154 pointy_brackets,155 use_index_order,156 float_format,157 double_format,158 use_field_number,159 descriptor_pool,160 message_formatter,161 print_unknown_fields=print_unknown_fields,162 force_colon=force_colon)163 printer.PrintMessage(message)164 result = out.getvalue()165 out.close()166 if as_one_line:167 return result.rstrip()168 return result169 170 171def MessageToBytes(message, **kwargs) -> bytes:172 """Convert protobuf message to encoded text format. See MessageToString."""173 text = MessageToString(message, **kwargs)174 if isinstance(text, bytes):175 return text176 codec = 'utf-8' if kwargs.get('as_utf8') else 'ascii'177 return text.encode(codec)178 179 180def _IsMapEntry(field):181 return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and182 field.message_type.has_options and183 field.message_type.GetOptions().map_entry)184 185 186def PrintMessage(message,187 out,188 indent=0,189 as_utf8=False,190 as_one_line=False,191 use_short_repeated_primitives=False,192 pointy_brackets=False,193 use_index_order=False,194 float_format=None,195 double_format=None,196 use_field_number=False,197 descriptor_pool=None,198 message_formatter=None,199 print_unknown_fields=False,200 force_colon=False):201 """Convert the message to text format and write it to the out stream.202 203 Args:204 message: The Message object to convert to text format.205 out: A file handle to write the message to.206 indent: The initial indent level for pretty print.207 as_utf8: Return unescaped Unicode for non-ASCII characters.208 as_one_line: Don't introduce newlines between fields.209 use_short_repeated_primitives: Use short repeated format for primitives.210 pointy_brackets: If True, use angle brackets instead of curly braces for211 nesting.212 use_index_order: If True, print fields of a proto message using the order213 defined in source code instead of the field number. By default, use the214 field number order.215 float_format: If set, use this to specify float field formatting216 (per the "Format Specification Mini-Language"); otherwise, shortest217 float that has same value in wire will be printed. Also affect double218 field if double_format is not set but float_format is set.219 double_format: If set, use this to specify double field formatting220 (per the "Format Specification Mini-Language"); if it is not set but221 float_format is set, use float_format. Otherwise, str() is used.222 use_field_number: If True, print field numbers instead of names.223 descriptor_pool: A DescriptorPool used to resolve Any types.224 message_formatter: A function(message, indent, as_one_line): unicode|None225 to custom format selected sub-messages (usually based on message type).226 Use to pretty print parts of the protobuf for easier diffing.227 print_unknown_fields: If True, unknown fields will be printed.228 force_colon: If set, a colon will be added after the field name even if229 the field is a proto message.230 """231 printer = _Printer(232 out=out, indent=indent, as_utf8=as_utf8,233 as_one_line=as_one_line,234 use_short_repeated_primitives=use_short_repeated_primitives,235 pointy_brackets=pointy_brackets,236 use_index_order=use_index_order,237 float_format=float_format,238 double_format=double_format,239 use_field_number=use_field_number,240 descriptor_pool=descriptor_pool,241 message_formatter=message_formatter,242 print_unknown_fields=print_unknown_fields,243 force_colon=force_colon)244 printer.PrintMessage(message)245 246 247def PrintField(field,248 value,249 out,250 indent=0,251 as_utf8=False,252 as_one_line=False,253 use_short_repeated_primitives=False,254 pointy_brackets=False,255 use_index_order=False,256 float_format=None,257 double_format=None,258 message_formatter=None,259 print_unknown_fields=False,260 force_colon=False):261 """Print a single field name/value pair."""262 printer = _Printer(out, indent, as_utf8, as_one_line,263 use_short_repeated_primitives, pointy_brackets,264 use_index_order, float_format, double_format,265 message_formatter=message_formatter,266 print_unknown_fields=print_unknown_fields,267 force_colon=force_colon)268 printer.PrintField(field, value)269 270 271def PrintFieldValue(field,272 value,273 out,274 indent=0,275 as_utf8=False,276 as_one_line=False,277 use_short_repeated_primitives=False,278 pointy_brackets=False,279 use_index_order=False,280 float_format=None,281 double_format=None,282 message_formatter=None,283 print_unknown_fields=False,284 force_colon=False):285 """Print a single field value (not including name)."""286 printer = _Printer(out, indent, as_utf8, as_one_line,287 use_short_repeated_primitives, pointy_brackets,288 use_index_order, float_format, double_format,289 message_formatter=message_formatter,290 print_unknown_fields=print_unknown_fields,291 force_colon=force_colon)292 printer.PrintFieldValue(field, value)293 294 295def _BuildMessageFromTypeName(type_name, descriptor_pool):296 """Returns a protobuf message instance.297 298 Args:299 type_name: Fully-qualified protobuf message type name string.300 descriptor_pool: DescriptorPool instance.301 302 Returns:303 A Message instance of type matching type_name, or None if the a Descriptor304 wasn't found matching type_name.305 """306 # pylint: disable=g-import-not-at-top307 if descriptor_pool is None:308 from google.protobuf import descriptor_pool as pool_mod309 descriptor_pool = pool_mod.Default()310 from google.protobuf import message_factory311 try:312 message_descriptor = descriptor_pool.FindMessageTypeByName(type_name)313 except KeyError:314 return None315 message_type = message_factory.GetMessageClass(message_descriptor)316 return message_type()317 318 319# These values must match WireType enum in //google/protobuf/wire_format.h.320WIRETYPE_LENGTH_DELIMITED = 2321WIRETYPE_START_GROUP = 3322 323 324class _Printer(object):325 """Text format printer for protocol message."""326 327 def __init__(328 self,329 out,330 indent=0,331 as_utf8=False,332 as_one_line=False,333 use_short_repeated_primitives=False,334 pointy_brackets=False,335 use_index_order=False,336 float_format=None,337 double_format=None,338 use_field_number=False,339 descriptor_pool=None,340 message_formatter=None,341 print_unknown_fields=False,342 force_colon=False):343 """Initialize the Printer.344 345 Double values can be formatted compactly with 15 digits of precision346 (which is the most that IEEE 754 "double" can guarantee) using347 double_format='.15g'. To ensure that converting to text and back to a proto348 will result in an identical value, double_format='.17g' should be used.349 350 Args:351 out: To record the text format result.352 indent: The initial indent level for pretty print.353 as_utf8: Return unescaped Unicode for non-ASCII characters.354 as_one_line: Don't introduce newlines between fields.355 use_short_repeated_primitives: Use short repeated format for primitives.356 pointy_brackets: If True, use angle brackets instead of curly braces for357 nesting.358 use_index_order: If True, print fields of a proto message using the order359 defined in source code instead of the field number. By default, use the360 field number order.361 float_format: If set, use this to specify float field formatting362 (per the "Format Specification Mini-Language"); otherwise, shortest363 float that has same value in wire will be printed. Also affect double364 field if double_format is not set but float_format is set.365 double_format: If set, use this to specify double field formatting366 (per the "Format Specification Mini-Language"); if it is not set but367 float_format is set, use float_format. Otherwise, str() is used.368 use_field_number: If True, print field numbers instead of names.369 descriptor_pool: A DescriptorPool used to resolve Any types.370 message_formatter: A function(message, indent, as_one_line): unicode|None371 to custom format selected sub-messages (usually based on message type).372 Use to pretty print parts of the protobuf for easier diffing.373 print_unknown_fields: If True, unknown fields will be printed.374 force_colon: If set, a colon will be added after the field name even if375 the field is a proto message.376 """377 self.out = out378 self.indent = indent379 self.as_utf8 = as_utf8380 self.as_one_line = as_one_line381 self.use_short_repeated_primitives = use_short_repeated_primitives382 self.pointy_brackets = pointy_brackets383 self.use_index_order = use_index_order384 self.float_format = float_format385 if double_format is not None:386 self.double_format = double_format387 else:388 self.double_format = float_format389 self.use_field_number = use_field_number390 self.descriptor_pool = descriptor_pool391 self.message_formatter = message_formatter392 self.print_unknown_fields = print_unknown_fields393 self.force_colon = force_colon394 395 def _TryPrintAsAnyMessage(self, message):396 """Serializes if message is a google.protobuf.Any field."""397 if '/' not in message.type_url:398 return False399 packed_message = _BuildMessageFromTypeName(message.TypeName(),400 self.descriptor_pool)401 if packed_message:402 packed_message.MergeFromString(message.value)403 colon = ':' if self.force_colon else ''404 self.out.write('%s[%s]%s ' % (self.indent * ' ', message.type_url, colon))405 self._PrintMessageFieldValue(packed_message)406 self.out.write(' ' if self.as_one_line else '\n')407 return True408 else:409 return False410 411 def _TryCustomFormatMessage(self, message):412 formatted = self.message_formatter(message, self.indent, self.as_one_line)413 if formatted is None:414 return False415 416 out = self.out417 out.write(' ' * self.indent)418 out.write(formatted)419 out.write(' ' if self.as_one_line else '\n')420 return True421 422 def PrintMessage(self, message):423 """Convert protobuf message to text format.424 425 Args:426 message: The protocol buffers message.427 """428 if self.message_formatter and self._TryCustomFormatMessage(message):429 return430 if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and431 self._TryPrintAsAnyMessage(message)):432 return433 fields = message.ListFields()434 if self.use_index_order:435 fields.sort(436 key=lambda x: x[0].number if x[0].is_extension else x[0].index)437 for field, value in fields:438 if _IsMapEntry(field):439 for key in sorted(value):440 # This is slow for maps with submessage entries because it copies the441 # entire tree. Unfortunately this would take significant refactoring442 # of this file to work around.443 #444 # TODO: refactor and optimize if this becomes an issue.445 entry_submsg = value.GetEntryClass()(key=key, value=value[key])446 self.PrintField(field, entry_submsg)447 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:448 if (self.use_short_repeated_primitives449 and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE450 and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_STRING):451 self._PrintShortRepeatedPrimitivesValue(field, value)452 else:453 for element in value:454 self.PrintField(field, element)455 else:456 self.PrintField(field, value)457 458 if self.print_unknown_fields:459 self._PrintUnknownFields(unknown_fields.UnknownFieldSet(message))460 461 def _PrintUnknownFields(self, unknown_field_set):462 """Print unknown fields."""463 out = self.out464 for field in unknown_field_set:465 out.write(' ' * self.indent)466 out.write(str(field.field_number))467 if field.wire_type == WIRETYPE_START_GROUP:468 if self.as_one_line:469 out.write(' { ')470 else:471 out.write(' {\n')472 self.indent += 2473 474 self._PrintUnknownFields(field.data)475 476 if self.as_one_line:477 out.write('} ')478 else:479 self.indent -= 2480 out.write(' ' * self.indent + '}\n')481 elif field.wire_type == WIRETYPE_LENGTH_DELIMITED:482 try:483 # If this field is parseable as a Message, it is probably484 # an embedded message.485 # pylint: disable=protected-access486 (embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet(487 memoryview(field.data), 0, len(field.data))488 except Exception: # pylint: disable=broad-except489 pos = 0490 491 if pos == len(field.data):492 if self.as_one_line:493 out.write(' { ')494 else:495 out.write(' {\n')496 self.indent += 2497 498 self._PrintUnknownFields(embedded_unknown_message)499 500 if self.as_one_line:501 out.write('} ')502 else:503 self.indent -= 2504 out.write(' ' * self.indent + '}\n')505 else:506 # A string or bytes field. self.as_utf8 may not work.507 out.write(': \"')508 out.write(text_encoding.CEscape(field.data, False))509 out.write('\" ' if self.as_one_line else '\"\n')510 else:511 # varint, fixed32, fixed64512 out.write(': ')513 out.write(str(field.data))514 out.write(' ' if self.as_one_line else '\n')515 516 def _PrintFieldName(self, field):517 """Print field name."""518 out = self.out519 out.write(' ' * self.indent)520 if self.use_field_number:521 out.write(str(field.number))522 else:523 if field.is_extension:524 out.write('[')525 if (field.containing_type.GetOptions().message_set_wire_format and526 field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and527 field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):528 out.write(field.message_type.full_name)529 else:530 out.write(field.full_name)531 out.write(']')532 elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:533 # For groups, use the capitalized name.534 out.write(field.message_type.name)535 else:536 out.write(field.name)537 538 if (self.force_colon or539 field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE):540 # The colon is optional in this case, but our cross-language golden files541 # don't include it. Here, the colon is only included if force_colon is542 # set to True543 out.write(':')544 545 def PrintField(self, field, value):546 """Print a single field name/value pair."""547 self._PrintFieldName(field)548 self.out.write(' ')549 self.PrintFieldValue(field, value)550 self.out.write(' ' if self.as_one_line else '\n')551 552 def _PrintShortRepeatedPrimitivesValue(self, field, value):553 """"Prints short repeated primitives value."""554 # Note: this is called only when value has at least one element.555 self._PrintFieldName(field)556 self.out.write(' [')557 for i in range(len(value) - 1):558 self.PrintFieldValue(field, value[i])559 self.out.write(', ')560 self.PrintFieldValue(field, value[-1])561 self.out.write(']')562 self.out.write(' ' if self.as_one_line else '\n')563 564 def _PrintMessageFieldValue(self, value):565 if self.pointy_brackets:566 openb = '<'567 closeb = '>'568 else:569 openb = '{'570 closeb = '}'571 572 if self.as_one_line:573 self.out.write('%s ' % openb)574 self.PrintMessage(value)575 self.out.write(closeb)576 else:577 self.out.write('%s\n' % openb)578 self.indent += 2579 self.PrintMessage(value)580 self.indent -= 2581 self.out.write(' ' * self.indent + closeb)582 583 def PrintFieldValue(self, field, value):584 """Print a single field value (not including name).585 586 For repeated fields, the value should be a single element.587 588 Args:589 field: The descriptor of the field to be printed.590 value: The value of the field.591 """592 out = self.out593 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:594 self._PrintMessageFieldValue(value)595 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:596 enum_value = field.enum_type.values_by_number.get(value, None)597 if enum_value is not None:598 out.write(enum_value.name)599 else:600 out.write(str(value))601 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:602 out.write('\"')603 if isinstance(value, str) and not self.as_utf8:604 out_value = value.encode('utf-8')605 else:606 out_value = value607 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:608 # We always need to escape all binary data in TYPE_BYTES fields.609 out_as_utf8 = False610 else:611 out_as_utf8 = self.as_utf8612 out.write(text_encoding.CEscape(out_value, out_as_utf8))613 out.write('\"')614 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:615 if value:616 out.write('true')617 else:618 out.write('false')619 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:620 if self.float_format is not None:621 out.write('{1:{0}}'.format(self.float_format, value))622 else:623 if math.isnan(value):624 out.write(str(value))625 else:626 out.write(str(type_checkers.ToShortestFloat(value)))627 elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_DOUBLE and628 self.double_format is not None):629 out.write('{1:{0}}'.format(self.double_format, value))630 else:631 out.write(str(value))632 633 634def Parse(text,635 message,636 allow_unknown_extension=False,637 allow_field_number=False,638 descriptor_pool=None,639 allow_unknown_field=False):640 """Parses a text representation of a protocol message into a message.641 642 NOTE: for historical reasons this function does not clear the input643 message. This is different from what the binary msg.ParseFrom(...) does.644 If text contains a field already set in message, the value is appended if the645 field is repeated. Otherwise, an error is raised.646 647 Example::648 649 a = MyProto()650 a.repeated_field.append('test')651 b = MyProto()652 653 # Repeated fields are combined654 text_format.Parse(repr(a), b)655 text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"]656 657 # Non-repeated fields cannot be overwritten658 a.singular_field = 1659 b.singular_field = 2660 text_format.Parse(repr(a), b) # ParseError661 662 # Binary version:663 b.ParseFromString(a.SerializeToString()) # repeated_field is now "test"664 665 Caller is responsible for clearing the message as needed.666 667 Args:668 text (str): Message text representation.669 message (Message): A protocol buffer message to merge into.670 allow_unknown_extension: if True, skip over missing extensions and keep671 parsing672 allow_field_number: if True, both field number and field name are allowed.673 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.674 allow_unknown_field: if True, skip over unknown field and keep675 parsing. Avoid to use this option if possible. It may hide some676 errors (e.g. spelling error on field name)677 678 Returns:679 Message: The same message passed as argument.680 681 Raises:682 ParseError: On text parsing problems.683 """684 return ParseLines(text.split(b'\n' if isinstance(text, bytes) else u'\n'),685 message,686 allow_unknown_extension,687 allow_field_number,688 descriptor_pool=descriptor_pool,689 allow_unknown_field=allow_unknown_field)690 691 692def Merge(text,693 message,694 allow_unknown_extension=False,695 allow_field_number=False,696 descriptor_pool=None,697 allow_unknown_field=False):698 """Parses a text representation of a protocol message into a message.699 700 Like Parse(), but allows repeated values for a non-repeated field, and uses701 the last one. This means any non-repeated, top-level fields specified in text702 replace those in the message.703 704 Args:705 text (str): Message text representation.706 message (Message): A protocol buffer message to merge into.707 allow_unknown_extension: if True, skip over missing extensions and keep708 parsing709 allow_field_number: if True, both field number and field name are allowed.710 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.711 allow_unknown_field: if True, skip over unknown field and keep712 parsing. Avoid to use this option if possible. It may hide some713 errors (e.g. spelling error on field name)714 715 Returns:716 Message: The same message passed as argument.717 718 Raises:719 ParseError: On text parsing problems.720 """721 return MergeLines(722 text.split(b'\n' if isinstance(text, bytes) else u'\n'),723 message,724 allow_unknown_extension,725 allow_field_number,726 descriptor_pool=descriptor_pool,727 allow_unknown_field=allow_unknown_field)728 729 730def ParseLines(lines,731 message,732 allow_unknown_extension=False,733 allow_field_number=False,734 descriptor_pool=None,735 allow_unknown_field=False):736 """Parses a text representation of a protocol message into a message.737 738 See Parse() for caveats.739 740 Args:741 lines: An iterable of lines of a message's text representation.742 message: A protocol buffer message to merge into.743 allow_unknown_extension: if True, skip over missing extensions and keep744 parsing745 allow_field_number: if True, both field number and field name are allowed.746 descriptor_pool: A DescriptorPool used to resolve Any types.747 allow_unknown_field: if True, skip over unknown field and keep748 parsing. Avoid to use this option if possible. It may hide some749 errors (e.g. spelling error on field name)750 751 Returns:752 The same message passed as argument.753 754 Raises:755 ParseError: On text parsing problems.756 """757 parser = _Parser(allow_unknown_extension,758 allow_field_number,759 descriptor_pool=descriptor_pool,760 allow_unknown_field=allow_unknown_field)761 return parser.ParseLines(lines, message)762 763 764def MergeLines(lines,765 message,766 allow_unknown_extension=False,767 allow_field_number=False,768 descriptor_pool=None,769 allow_unknown_field=False):770 """Parses a text representation of a protocol message into a message.771 772 See Merge() for more details.773 774 Args:775 lines: An iterable of lines of a message's text representation.776 message: A protocol buffer message to merge into.777 allow_unknown_extension: if True, skip over missing extensions and keep778 parsing779 allow_field_number: if True, both field number and field name are allowed.780 descriptor_pool: A DescriptorPool used to resolve Any types.781 allow_unknown_field: if True, skip over unknown field and keep782 parsing. Avoid to use this option if possible. It may hide some783 errors (e.g. spelling error on field name)784 785 Returns:786 The same message passed as argument.787 788 Raises:789 ParseError: On text parsing problems.790 """791 parser = _Parser(allow_unknown_extension,792 allow_field_number,793 descriptor_pool=descriptor_pool,794 allow_unknown_field=allow_unknown_field)795 return parser.MergeLines(lines, message)796 797 798class _Parser(object):799 """Text format parser for protocol message."""800 801 def __init__(self,802 allow_unknown_extension=False,803 allow_field_number=False,804 descriptor_pool=None,805 allow_unknown_field=False):806 self.allow_unknown_extension = allow_unknown_extension807 self.allow_field_number = allow_field_number808 self.descriptor_pool = descriptor_pool809 self.allow_unknown_field = allow_unknown_field810 811 def ParseLines(self, lines, message):812 """Parses a text representation of a protocol message into a message."""813 self._allow_multiple_scalars = False814 self._ParseOrMerge(lines, message)815 return message816 817 def MergeLines(self, lines, message):818 """Merges a text representation of a protocol message into a message."""819 self._allow_multiple_scalars = True820 self._ParseOrMerge(lines, message)821 return message822 823 def _ParseOrMerge(self, lines, message):824 """Converts a text representation of a protocol message into a message.825 826 Args:827 lines: Lines of a message's text representation.828 message: A protocol buffer message to merge into.829 830 Raises:831 ParseError: On text parsing problems.832 """833 # Tokenize expects native str lines.834 try:835 str_lines = (836 line if isinstance(line, str) else line.decode('utf-8')837 for line in lines)838 tokenizer = Tokenizer(str_lines)839 except UnicodeDecodeError as e:840 raise ParseError from e841 if message:842 self.root_type = message.DESCRIPTOR.full_name843 while not tokenizer.AtEnd():844 self._MergeField(tokenizer, message)845 846 def _MergeField(self, tokenizer, message):847 """Merges a single protocol message field into a message.848 849 Args:850 tokenizer: A tokenizer to parse the field name and values.851 message: A protocol message to record the data.852 853 Raises:854 ParseError: In case of text parsing problems.855 """856 message_descriptor = message.DESCRIPTOR857 if (message_descriptor.full_name == _ANY_FULL_TYPE_NAME and858 tokenizer.TryConsume('[')):859 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)860 tokenizer.Consume(']')861 tokenizer.TryConsume(':')862 self._DetectSilentMarker(tokenizer, message_descriptor.full_name,863 type_url_prefix + '/' + packed_type_name)864 if tokenizer.TryConsume('<'):865 expanded_any_end_token = '>'866 else:867 tokenizer.Consume('{')868 expanded_any_end_token = '}'869 expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,870 self.descriptor_pool)871 # Direct comparison with None is used instead of implicit bool conversion872 # to avoid false positives with falsy initial values, e.g. for873 # google.protobuf.ListValue.874 if expanded_any_sub_message is None:875 raise ParseError('Type %s not found in descriptor pool' %876 packed_type_name)877 while not tokenizer.TryConsume(expanded_any_end_token):878 if tokenizer.AtEnd():879 raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %880 (expanded_any_end_token,))881 self._MergeField(tokenizer, expanded_any_sub_message)882 deterministic = False883 884 message.Pack(expanded_any_sub_message,885 type_url_prefix=type_url_prefix,886 deterministic=deterministic)887 return888 889 if tokenizer.TryConsume('['):890 name = [tokenizer.ConsumeIdentifier()]891 while tokenizer.TryConsume('.'):892 name.append(tokenizer.ConsumeIdentifier())893 name = '.'.join(name)894 895 if not message_descriptor.is_extendable:896 raise tokenizer.ParseErrorPreviousToken(897 'Message type "%s" does not have extensions.' %898 message_descriptor.full_name)899 # pylint: disable=protected-access900 field = message.Extensions._FindExtensionByName(name)901 # pylint: enable=protected-access902 if not field:903 if self.allow_unknown_extension:904 field = None905 else:906 raise tokenizer.ParseErrorPreviousToken(907 'Extension "%s" not registered. '908 'Did you import the _pb2 module which defines it? '909 'If you are trying to place the extension in the MessageSet '910 'field of another message that is in an Any or MessageSet field, '911 'that message\'s _pb2 module must be imported as well' % name)912 elif message_descriptor != field.containing_type:913 raise tokenizer.ParseErrorPreviousToken(914 'Extension "%s" does not extend message type "%s".' %915 (name, message_descriptor.full_name))916 917 tokenizer.Consume(']')918 919 else:920 name = tokenizer.ConsumeIdentifierOrNumber()921 if self.allow_field_number and name.isdigit():922 number = ParseInteger(name, True, True)923 field = message_descriptor.fields_by_number.get(number, None)924 if not field and message_descriptor.is_extendable:925 field = message.Extensions._FindExtensionByNumber(number)926 else:927 field = message_descriptor.fields_by_name.get(name, None)928 929 # Group names are expected to be capitalized as they appear in the930 # .proto file, which actually matches their type names, not their field931 # names.932 if not field:933 field = message_descriptor.fields_by_name.get(name.lower(), None)934 if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP:935 field = None936 937 if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and938 field.message_type.name != name):939 field = None940 941 if not field and not self.allow_unknown_field:942 raise tokenizer.ParseErrorPreviousToken(943 'Message type "%s" has no field named "%s".' %944 (message_descriptor.full_name, name))945 946 if field:947 if not self._allow_multiple_scalars and field.containing_oneof:948 # Check if there's a different field set in this oneof.949 # Note that we ignore the case if the same field was set before, and we950 # apply _allow_multiple_scalars to non-scalar fields as well.951 which_oneof = message.WhichOneof(field.containing_oneof.name)952 if which_oneof is not None and which_oneof != field.name:953 raise tokenizer.ParseErrorPreviousToken(954 'Field "%s" is specified along with field "%s", another member '955 'of oneof "%s" for message type "%s".' %956 (field.name, which_oneof, field.containing_oneof.name,957 message_descriptor.full_name))958 959 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:960 tokenizer.TryConsume(':')961 self._DetectSilentMarker(tokenizer, message_descriptor.full_name,962 field.full_name)963 merger = self._MergeMessageField964 else:965 tokenizer.Consume(':')966 self._DetectSilentMarker(tokenizer, message_descriptor.full_name,967 field.full_name)968 merger = self._MergeScalarField969 970 if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and971 tokenizer.TryConsume('[')):972 # Short repeated format, e.g. "foo: [1, 2, 3]"973 if not tokenizer.TryConsume(']'):974 while True:975 merger(tokenizer, message, field)976 if tokenizer.TryConsume(']'):977 break978 tokenizer.Consume(',')979 980 else:981 merger(tokenizer, message, field)982 983 else: # Proto field is unknown.984 assert (self.allow_unknown_extension or self.allow_unknown_field)985 self._SkipFieldContents(tokenizer, name, message_descriptor.full_name)986 987 # For historical reasons, fields may optionally be separated by commas or988 # semicolons.989 if not tokenizer.TryConsume(','):990 tokenizer.TryConsume(';')991 992 def _LogSilentMarker(self, immediate_message_type, field_name):993 pass994 995 def _DetectSilentMarker(self, tokenizer, immediate_message_type, field_name):996 if tokenizer.contains_silent_marker_before_current_token:997 self._LogSilentMarker(immediate_message_type, field_name)998 999 def _ConsumeAnyTypeUrl(self, tokenizer):1000 """Consumes a google.protobuf.Any type URL and returns the type name."""1001 # Consume "type.googleapis.com/".1002 prefix = [tokenizer.ConsumeIdentifier()]1003 tokenizer.Consume('.')1004 prefix.append(tokenizer.ConsumeIdentifier())1005 tokenizer.Consume('.')1006 prefix.append(tokenizer.ConsumeIdentifier())1007 tokenizer.Consume('/')1008 # Consume the fully-qualified type name.1009 name = [tokenizer.ConsumeIdentifier()]1010 while tokenizer.TryConsume('.'):1011 name.append(tokenizer.ConsumeIdentifier())1012 return '.'.join(prefix), '.'.join(name)1013 1014 def _MergeMessageField(self, tokenizer, message, field):1015 """Merges a single scalar field into a message.1016 1017 Args:1018 tokenizer: A tokenizer to parse the field value.1019 message: The message of which field is a member.1020 field: The descriptor of the field to be merged.1021 1022 Raises:1023 ParseError: In case of text parsing problems.1024 """1025 is_map_entry = _IsMapEntry(field)1026 1027 if tokenizer.TryConsume('<'):1028 end_token = '>'1029 else:1030 tokenizer.Consume('{')1031 end_token = '}'1032 1033 if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:1034 if field.is_extension:1035 sub_message = message.Extensions[field].add()1036 elif is_map_entry:1037 sub_message = getattr(message, field.name).GetEntryClass()()1038 else:1039 sub_message = getattr(message, field.name).add()1040 else:1041 if field.is_extension:1042 if (not self._allow_multiple_scalars and1043 message.HasExtension(field)):1044 raise tokenizer.ParseErrorPreviousToken(1045 'Message type "%s" should not have multiple "%s" extensions.' %1046 (message.DESCRIPTOR.full_name, field.full_name))1047 sub_message = message.Extensions[field]1048 else:1049 # Also apply _allow_multiple_scalars to message field.1050 # TODO: Change to _allow_singular_overwrites.1051 if (not self._allow_multiple_scalars and1052 message.HasField(field.name)):1053 raise tokenizer.ParseErrorPreviousToken(1054 'Message type "%s" should not have multiple "%s" fields.' %1055 (message.DESCRIPTOR.full_name, field.name))1056 sub_message = getattr(message, field.name)1057 sub_message.SetInParent()1058 1059 while not tokenizer.TryConsume(end_token):1060 if tokenizer.AtEnd():1061 raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,))1062 self._MergeField(tokenizer, sub_message)1063 1064 if is_map_entry:1065 value_cpptype = field.message_type.fields_by_name['value'].cpp_type1066 if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:1067 value = getattr(message, field.name)[sub_message.key]1068 value.CopyFrom(sub_message.value)1069 else:1070 getattr(message, field.name)[sub_message.key] = sub_message.value1071 1072 def _MergeScalarField(self, tokenizer, message, field):1073 """Merges a single scalar field into a message.1074 1075 Args:1076 tokenizer: A tokenizer to parse the field value.1077 message: A protocol message to record the data.1078 field: The descriptor of the field to be merged.1079 1080 Raises:1081 ParseError: In case of text parsing problems.1082 RuntimeError: On runtime errors.1083 """1084 _ = self.allow_unknown_extension1085 value = None1086 1087 if field.type in (descriptor.FieldDescriptor.TYPE_INT32,1088 descriptor.FieldDescriptor.TYPE_SINT32,1089 descriptor.FieldDescriptor.TYPE_SFIXED32):1090 value = _ConsumeInt32(tokenizer)1091 elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,1092 descriptor.FieldDescriptor.TYPE_SINT64,1093 descriptor.FieldDescriptor.TYPE_SFIXED64):1094 value = _ConsumeInt64(tokenizer)1095 elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,1096 descriptor.FieldDescriptor.TYPE_FIXED32):1097 value = _ConsumeUint32(tokenizer)1098 elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,1099 descriptor.FieldDescriptor.TYPE_FIXED64):1100 value = _ConsumeUint64(tokenizer)1101 elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,1102 descriptor.FieldDescriptor.TYPE_DOUBLE):1103 value = tokenizer.ConsumeFloat()1104 elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:1105 value = tokenizer.ConsumeBool()1106 elif field.type == descriptor.FieldDescriptor.TYPE_STRING:1107 value = tokenizer.ConsumeString()1108 elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:1109 value = tokenizer.ConsumeByteString()1110 elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:1111 value = tokenizer.ConsumeEnum(field)1112 else:1113 raise RuntimeError('Unknown field type %d' % field.type)1114 1115 if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:1116 if field.is_extension:1117 message.Extensions[field].append(value)1118 else:1119 getattr(message, field.name).append(value)1120 else:1121 if field.is_extension:1122 if (not self._allow_multiple_scalars and1123 field.has_presence and1124 message.HasExtension(field)):1125 raise tokenizer.ParseErrorPreviousToken(1126 'Message type "%s" should not have multiple "%s" extensions.' %1127 (message.DESCRIPTOR.full_name, field.full_name))1128 else:1129 message.Extensions[field] = value1130 else:1131 duplicate_error = False1132 if not self._allow_multiple_scalars:1133 if field.has_presence:1134 duplicate_error = message.HasField(field.name)1135 else:1136 # For field that doesn't represent presence, try best effort to1137 # check multiple scalars by compare to default values.1138 duplicate_error = bool(getattr(message, field.name))1139 1140 if duplicate_error:1141 raise tokenizer.ParseErrorPreviousToken(1142 'Message type "%s" should not have multiple "%s" fields.' %1143 (message.DESCRIPTOR.full_name, field.name))1144 else:1145 setattr(message, field.name, value)1146 1147 def _SkipFieldContents(self, tokenizer, field_name, immediate_message_type):1148 """Skips over contents (value or message) of a field.1149 1150 Args:1151 tokenizer: A tokenizer to parse the field name and values.1152 field_name: The field name currently being parsed.1153 immediate_message_type: The type of the message immediately containing1154 the silent marker.1155 """1156 # Try to guess the type of this field.1157 # If this field is not a message, there should be a ":" between the1158 # field name and the field value and also the field value should not1159 # start with "{" or "<" which indicates the beginning of a message body.1160 # If there is no ":" or there is a "{" or "<" after ":", this field has1161 # to be a message or the input is ill-formed.1162 if tokenizer.TryConsume(1163 ':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'):1164 self._DetectSilentMarker(tokenizer, immediate_message_type, field_name)1165 if tokenizer.LookingAt('['):1166 self._SkipRepeatedFieldValue(tokenizer)1167 else:1168 self._SkipFieldValue(tokenizer)1169 else:1170 self._DetectSilentMarker(tokenizer, immediate_message_type, field_name)1171 self._SkipFieldMessage(tokenizer, immediate_message_type)1172 1173 def _SkipField(self, tokenizer, immediate_message_type):1174 """Skips over a complete field (name and value/message).1175 1176 Args:1177 tokenizer: A tokenizer to parse the field name and values.1178 immediate_message_type: The type of the message immediately containing1179 the silent marker.1180 """1181 field_name = ''1182 if tokenizer.TryConsume('['):1183 # Consume extension or google.protobuf.Any type URL1184 field_name += '[' + tokenizer.ConsumeIdentifier()1185 num_identifiers = 11186 while tokenizer.TryConsume('.'):1187 field_name += '.' + tokenizer.ConsumeIdentifier()1188 num_identifiers += 11189 # This is possibly a type URL for an Any message.1190 if num_identifiers == 3 and tokenizer.TryConsume('/'):1191 field_name += '/' + tokenizer.ConsumeIdentifier()1192 while tokenizer.TryConsume('.'):1193 field_name += '.' + tokenizer.ConsumeIdentifier()1194 tokenizer.Consume(']')1195 field_name += ']'1196 else:1197 field_name += tokenizer.ConsumeIdentifierOrNumber()1198 1199 self._SkipFieldContents(tokenizer, field_name, immediate_message_type)1200