codekingpro/portable-devtools

1# Protocol Buffers - Google's data interchange format2# Copyright 2008 Google Inc.  All rights reserved.3#4# Use of this source code is governed by a BSD-style5# license that can be found in the LICENSE file or at6# https://developers.google.com/open-source/licenses/bsd7 8"""Contains routines for printing protocol messages in JSON format.9 10Simple usage example:11 12  # Create a proto object and serialize it to a json format string.13  message = my_proto_pb2.MyMessage(foo='bar')14  json_string = json_format.MessageToJson(message)15 16  # Parse a json format string to proto object.17  message = json_format.Parse(json_string, my_proto_pb2.MyMessage())18"""19 20__author__ = 'jieluo@google.com (Jie Luo)'21 22 23import base6424from collections import OrderedDict25import json26import math27from operator import methodcaller28import re29import warnings30 31from google.protobuf import descriptor32from google.protobuf import message_factory33from google.protobuf import symbol_database34from google.protobuf.internal import type_checkers35 36 37_INT_TYPES = frozenset([38    descriptor.FieldDescriptor.CPPTYPE_INT32,39    descriptor.FieldDescriptor.CPPTYPE_UINT32,40    descriptor.FieldDescriptor.CPPTYPE_INT64,41    descriptor.FieldDescriptor.CPPTYPE_UINT64,42])43_INT64_TYPES = frozenset([44    descriptor.FieldDescriptor.CPPTYPE_INT64,45    descriptor.FieldDescriptor.CPPTYPE_UINT64,46])47_FLOAT_TYPES = frozenset([48    descriptor.FieldDescriptor.CPPTYPE_FLOAT,49    descriptor.FieldDescriptor.CPPTYPE_DOUBLE,50])51_INFINITY = 'Infinity'52_NEG_INFINITY = '-Infinity'53_NAN = 'NaN'54 55_UNPAIRED_SURROGATE_PATTERN = re.compile(56    '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'57)58 59_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')60 61 62class Error(Exception):63  """Top-level module error for json_format."""64 65 66class SerializeToJsonError(Error):67  """Thrown if serialization to JSON fails."""68 69 70class ParseError(Error):71  """Thrown in case of parsing error."""72 73 74class EnumStringValueParseError(ParseError):75  """Thrown if unknown string enum value is encountered.76 77  This exception is suppressed if ignore_unknown_fields is set.78  """79 80 81def MessageToJson(82    message,83    preserving_proto_field_name=False,84    indent=2,85    sort_keys=False,86    use_integers_for_enums=False,87    descriptor_pool=None,88    float_precision=None,89    ensure_ascii=True,90    always_print_fields_with_no_presence=False,91):92  """Converts protobuf message to JSON format.93 94  Args:95    message: The protocol buffers message instance to serialize.96    always_print_fields_with_no_presence: If True, fields without presence97      (implicit presence scalars, repeated fields, and map fields) will always98      be serialized. Any field that supports presence is not affected by this99      option (including singular message fields and oneof fields).100    preserving_proto_field_name: If True, use the original proto field names as101      defined in the .proto file. If False, convert the field names to102      lowerCamelCase.103    indent: The JSON object will be pretty-printed with this indent level. An104      indent level of 0 or negative will only insert newlines. If the indent105      level is None, no newlines will be inserted.106    sort_keys: If True, then the output will be sorted by field names.107    use_integers_for_enums: If true, print integers instead of enum names.108    descriptor_pool: A Descriptor Pool for resolving types. If None use the109      default.110    float_precision: Deprecated. If set, use this to specify float field valid111      digits.112    ensure_ascii: If True, strings with non-ASCII characters are escaped. If113      False, Unicode strings are returned unchanged.114 115  Returns:116    A string containing the JSON formatted protocol buffer message.117  """118  printer = _Printer(119      preserving_proto_field_name,120      use_integers_for_enums,121      descriptor_pool,122      float_precision,123      always_print_fields_with_no_presence,124  )125  return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)126 127 128def MessageToDict(129    message,130    always_print_fields_with_no_presence=False,131    preserving_proto_field_name=False,132    use_integers_for_enums=False,133    descriptor_pool=None,134    float_precision=None,135):136  """Converts protobuf message to a dictionary.137 138  When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.139 140  Args:141    message: The protocol buffers message instance to serialize.142    always_print_fields_with_no_presence: If True, fields without presence143      (implicit presence scalars, repeated fields, and map fields) will always144      be serialized. Any field that supports presence is not affected by this145      option (including singular message fields and oneof fields).146    preserving_proto_field_name: If True, use the original proto field names as147      defined in the .proto file. If False, convert the field names to148      lowerCamelCase.149    use_integers_for_enums: If true, print integers instead of enum names.150    descriptor_pool: A Descriptor Pool for resolving types. If None use the151      default.152    float_precision: Deprecated. If set, use this to specify float field valid153      digits.154 155  Returns:156    A dict representation of the protocol buffer message.157  """158  printer = _Printer(159      preserving_proto_field_name,160      use_integers_for_enums,161      descriptor_pool,162      float_precision,163      always_print_fields_with_no_presence,164  )165  # pylint: disable=protected-access166  return printer._MessageToJsonObject(message)167 168 169def _IsMapEntry(field):170  return (171      field.type == descriptor.FieldDescriptor.TYPE_MESSAGE172      and field.message_type.has_options173      and field.message_type.GetOptions().map_entry174  )175 176 177class _Printer(object):178  """JSON format printer for protocol message."""179 180  def __init__(181      self,182      preserving_proto_field_name=False,183      use_integers_for_enums=False,184      descriptor_pool=None,185      float_precision=None,186      always_print_fields_with_no_presence=False,187  ):188    self.always_print_fields_with_no_presence = (189        always_print_fields_with_no_presence190    )191    self.preserving_proto_field_name = preserving_proto_field_name192    self.use_integers_for_enums = use_integers_for_enums193    self.descriptor_pool = descriptor_pool194    if float_precision:195      warnings.warn(196          'float_precision option is deprecated for json_format. '197          'This will turn into error in 7.34.0, please remove it '198          'before that.'199      )200      self.float_format = '.{}g'.format(float_precision)201    else:202      self.float_format = None203 204  def ToJsonString(self, message, indent, sort_keys, ensure_ascii):205    js = self._MessageToJsonObject(message)206    return json.dumps(207        js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii208    )209 210  def _MessageToJsonObject(self, message):211    """Converts message to an object according to Proto3 JSON Specification."""212    message_descriptor = message.DESCRIPTOR213    full_name = message_descriptor.full_name214    if _IsWrapperMessage(message_descriptor):215      return self._WrapperMessageToJsonObject(message)216    if full_name in _WKTJSONMETHODS:217      return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)218    js = {}219    return self._RegularMessageToJsonObject(message, js)220 221  def _RegularMessageToJsonObject(self, message, js):222    """Converts normal message according to Proto3 JSON Specification."""223    fields = message.ListFields()224 225    try:226      for field, value in fields:227        if field.is_extension:228          name = '[%s]' % field.full_name229        elif self.preserving_proto_field_name:230          name = field.name231        else:232          name = field.json_name233 234        if _IsMapEntry(field):235          # Convert a map field.236          v_field = field.message_type.fields_by_name['value']237          js_map = {}238          for key in value:239            if isinstance(key, bool):240              if key:241                recorded_key = 'true'242              else:243                recorded_key = 'false'244            else:245              recorded_key = str(key)246            js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])247          js[name] = js_map248        elif field.is_repeated:249          # Convert a repeated field.250          js[name] = [self._FieldToJsonObject(field, k) for k in value]251        else:252          js[name] = self._FieldToJsonObject(field, value)253 254      # Serialize default value if including_default_value_fields is True.255      if (256          self.always_print_fields_with_no_presence257      ):258        message_descriptor = message.DESCRIPTOR259        for field in message_descriptor.fields:260 261          # always_print_fields_with_no_presence doesn't apply to262          # any field which supports presence.263          if self.always_print_fields_with_no_presence and field.has_presence:264            continue265 266          if self.preserving_proto_field_name:267            name = field.name268          else:269            name = field.json_name270          if name in js:271            # Skip the field which has been serialized already.272            continue273          if _IsMapEntry(field):274            js[name] = {}275          elif field.is_repeated:276            js[name] = []277          else:278            js[name] = self._FieldToJsonObject(field, field.default_value)279 280    except ValueError as e:281      raise SerializeToJsonError(282          'Failed to serialize {0} field: {1}.'.format(field.name, e)283      ) from e284 285    return js286 287  def _FieldToJsonObject(self, field, value):288    """Converts field value according to Proto3 JSON Specification."""289    if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:290      return self._MessageToJsonObject(value)291    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:292      if self.use_integers_for_enums:293        return value294      if field.enum_type.full_name == 'google.protobuf.NullValue':295        return None296      enum_value = field.enum_type.values_by_number.get(value, None)297      if enum_value is not None:298        return enum_value.name299      else:300        if field.enum_type.is_closed:301          raise SerializeToJsonError(302              'Enum field contains an integer value '303              'which can not mapped to an enum value.'304          )305        else:306          return value307    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:308      if field.type == descriptor.FieldDescriptor.TYPE_BYTES:309        # Use base64 Data encoding for bytes310        return base64.b64encode(value).decode('utf-8')311      else:312        return str(value)313    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:314      return bool(value)315    elif field.cpp_type in _INT64_TYPES:316      return str(value)317    elif field.cpp_type in _FLOAT_TYPES:318      if math.isinf(value):319        if value < 0.0:320          return _NEG_INFINITY321        else:322          return _INFINITY323      if math.isnan(value):324        return _NAN325      if self.float_format:326        return float(format(value, self.float_format))327      elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:328        return type_checkers.ToShortestFloat(value)329 330    return value331 332  def _AnyMessageToJsonObject(self, message):333    """Converts Any message according to Proto3 JSON Specification."""334    if not message.ListFields():335      return {}336    # Must print @type first, use OrderedDict instead of {}337    js = OrderedDict()338    type_url = message.type_url339    js['@type'] = type_url340    sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)341    sub_message.ParseFromString(message.value)342    message_descriptor = sub_message.DESCRIPTOR343    full_name = message_descriptor.full_name344    if _IsWrapperMessage(message_descriptor):345      js['value'] = self._WrapperMessageToJsonObject(sub_message)346      return js347    if full_name in _WKTJSONMETHODS:348      js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(349          self350      )351      return js352    return self._RegularMessageToJsonObject(sub_message, js)353 354  def _GenericMessageToJsonObject(self, message):355    """Converts message according to Proto3 JSON Specification."""356    # Duration, Timestamp and FieldMask have ToJsonString method to do the357    # convert. Users can also call the method directly.358    return message.ToJsonString()359 360  def _ValueMessageToJsonObject(self, message):361    """Converts Value message according to Proto3 JSON Specification."""362    which = message.WhichOneof('kind')363    # If the Value message is not set treat as null_value when serialize364    # to JSON. The parse back result will be different from original message.365    if which is None or which == 'null_value':366      return None367    if which == 'list_value':368      return self._ListValueMessageToJsonObject(message.list_value)369    if which == 'number_value':370      value = message.number_value371      if math.isinf(value):372        raise ValueError(373            'Fail to serialize Infinity for Value.number_value, '374            'which would parse as string_value'375        )376      if math.isnan(value):377        raise ValueError(378            'Fail to serialize NaN for Value.number_value, '379            'which would parse as string_value'380        )381    else:382      value = getattr(message, which)383    oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]384    return self._FieldToJsonObject(oneof_descriptor, value)385 386  def _ListValueMessageToJsonObject(self, message):387    """Converts ListValue message according to Proto3 JSON Specification."""388    return [self._ValueMessageToJsonObject(value) for value in message.values]389 390  def _StructMessageToJsonObject(self, message):391    """Converts Struct message according to Proto3 JSON Specification."""392    fields = message.fields393    ret = {}394    for key in fields:395      ret[key] = self._ValueMessageToJsonObject(fields[key])396    return ret397 398  def _WrapperMessageToJsonObject(self, message):399    return self._FieldToJsonObject(400        message.DESCRIPTOR.fields_by_name['value'], message.value401    )402 403 404def _IsWrapperMessage(message_descriptor):405  return message_descriptor.file.name == 'google/protobuf/wrappers.proto'406 407 408def _DuplicateChecker(js):409  result = {}410  for name, value in js:411    if name in result:412      raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))413    result[name] = value414  return result415 416 417def _CreateMessageFromTypeUrl(type_url, descriptor_pool):418  """Creates a message from a type URL."""419  db = symbol_database.Default()420  pool = db.pool if descriptor_pool is None else descriptor_pool421  type_name = type_url.split('/')[-1]422  try:423    message_descriptor = pool.FindMessageTypeByName(type_name)424  except KeyError as e:425    raise TypeError(426        'Can not find message descriptor by type_url: {0}'.format(type_url)427    ) from e428  message_class = message_factory.GetMessageClass(message_descriptor)429  return message_class()430 431 432def Parse(433    text,434    message,435    ignore_unknown_fields=False,436    descriptor_pool=None,437    max_recursion_depth=100,438):439  """Parses a JSON representation of a protocol message into a message.440 441  Args:442    text: Message JSON representation.443    message: A protocol buffer message to merge into.444    ignore_unknown_fields: If True, do not raise errors for unknown fields.445    descriptor_pool: A Descriptor Pool for resolving types. If None use the446      default.447    max_recursion_depth: max recursion depth of JSON message to be deserialized.448      JSON messages over this depth will fail to be deserialized. Default value449      is 100.450 451  Returns:452    The same message passed as argument.453 454  Raises::455    ParseError: On JSON parsing problems.456  """457  if not isinstance(text, str):458    text = text.decode('utf-8')459 460  try:461    js = json.loads(text, object_pairs_hook=_DuplicateChecker)462  except Exception as e:463    raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e464 465  try:466    return ParseDict(467        js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth468    )469  except ParseError as e:470    raise e471  except Exception as e:472    raise ParseError(473        'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e))474    ) from e475 476 477def ParseDict(478    js_dict,479    message,480    ignore_unknown_fields=False,481    descriptor_pool=None,482    max_recursion_depth=100,483):484  """Parses a JSON dictionary representation into a message.485 486  Args:487    js_dict: Dict representation of a JSON message.488    message: A protocol buffer message to merge into.489    ignore_unknown_fields: If True, do not raise errors for unknown fields.490    descriptor_pool: A Descriptor Pool for resolving types. If None use the491      default.492    max_recursion_depth: max recursion depth of JSON message to be deserialized.493      JSON messages over this depth will fail to be deserialized. Default value494      is 100.495 496  Returns:497    The same message passed as argument.498  """499  parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)500  parser.ConvertMessage(js_dict, message, '')501  return message502 503 504_INT_OR_FLOAT = (int, float)505_LIST_LIKE = (list, tuple)506 507 508class _Parser(object):509  """JSON format parser for protocol message."""510 511  def __init__(512      self, ignore_unknown_fields, descriptor_pool, max_recursion_depth513  ):514    self.ignore_unknown_fields = ignore_unknown_fields515    self.descriptor_pool = descriptor_pool516    self.max_recursion_depth = max_recursion_depth517    self.recursion_depth = 0518 519  def ConvertMessage(self, value, message, path):520    """Convert a JSON object into a message.521 522    Args:523      value: A JSON object.524      message: A WKT or regular protocol message to record the data.525      path: parent path to log parse error info.526 527    Raises:528      ParseError: In case of convert problems.529    """530    # Increment recursion depth at message entry. The max_recursion_depth limit531    # is exclusive: a depth value equal to max_recursion_depth will trigger an532    # error. For example, with max_recursion_depth=5, nesting up to depth 4 is533    # allowed, but attempting depth 5 raises ParseError.534    self.recursion_depth += 1535    if self.recursion_depth > self.max_recursion_depth:536      raise ParseError(537          'Message too deep. Max recursion depth is {0}'.format(538              self.max_recursion_depth539          )540      )541    message_descriptor = message.DESCRIPTOR542    full_name = message_descriptor.full_name543    if not path:544      path = message_descriptor.name545    if _IsWrapperMessage(message_descriptor):546      self._ConvertWrapperMessage(value, message, path)547    elif full_name in _WKTJSONMETHODS:548      methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)549    else:550      self._ConvertFieldValuePair(value, message, path)551    self.recursion_depth -= 1552 553  def _ConvertFieldValuePair(self, js, message, path):554    """Convert field value pairs into regular message.555 556    Args:557      js: A JSON object to convert the field value pairs.558      message: A regular protocol message to record the data.559      path: parent path to log parse error info.560 561    Raises:562      ParseError: In case of problems converting.563    """564    names = []565    message_descriptor = message.DESCRIPTOR566    fields_by_json_name = dict(567        (f.json_name, f) for f in message_descriptor.fields568    )569 570    def _ClearFieldOrExtension(message, field):571      if field.is_extension:572        message.ClearExtension(field)573      else:574        message.ClearField(field.name)575 576    def _GetFieldOrExtension(message, field):577      if field.is_extension:578        return message.Extensions[field]579      else:580        return getattr(message, field.name)581 582    def _SetFieldOrExtension(message, field, value):583      if field.is_extension:584        message.Extensions[field] = value585      else:586        setattr(message, field.name, value)587 588    for name in js:589      try:590        field = fields_by_json_name.get(name, None)591        if not field:592          field = message_descriptor.fields_by_name.get(name, None)593        if not field and _VALID_EXTENSION_NAME.match(name):594          if not message_descriptor.is_extendable:595            raise ParseError(596                'Message type {0} does not have extensions at {1}'.format(597                    message_descriptor.full_name, path598                )599            )600          identifier = name[1:-1]  # strip [] brackets601          # pylint: disable=protected-access602          field = message.Extensions._FindExtensionByName(identifier)603          # pylint: enable=protected-access604          if not field:605            # Try looking for extension by the message type name, dropping the606            # field name following the final . separator in full_name.607            identifier = '.'.join(identifier.split('.')[:-1])608            # pylint: disable=protected-access609            field = message.Extensions._FindExtensionByName(identifier)610            # pylint: enable=protected-access611        if not field:612          if self.ignore_unknown_fields:613            continue614          raise ParseError(615              (616                  'Message type "{0}" has no field named "{1}" at "{2}".\n'617                  ' Available Fields(except extensions): "{3}"'618              ).format(619                  message_descriptor.full_name,620                  name,621                  path,622                  [f.json_name for f in message_descriptor.fields],623              )624          )625        if name in names:626          raise ParseError(627              'Message type "{0}" should not have multiple '628              '"{1}" fields at "{2}".'.format(629                  message.DESCRIPTOR.full_name, name, path630              )631          )632        names.append(name)633        value = js[name]634        # Check no other oneof field is parsed.635        if field.containing_oneof is not None and value is not None:636          oneof_name = field.containing_oneof.name637          if oneof_name in names:638            raise ParseError(639                'Message type "{0}" should not have multiple '640                '"{1}" oneof fields at "{2}".'.format(641                    message.DESCRIPTOR.full_name, oneof_name, path642                )643            )644          names.append(oneof_name)645 646        if value is None:647          if (648              field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE649              and field.message_type.full_name == 'google.protobuf.Value'650          ):651            sub_message = _GetFieldOrExtension(message, field)652            sub_message.null_value = 0653          elif (654              field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM655              and field.enum_type.full_name == 'google.protobuf.NullValue'656          ):657            _SetFieldOrExtension(message, field, 0)658          else:659            _ClearFieldOrExtension(message, field)660          continue661 662        # Parse field value.663        if _IsMapEntry(field):664          _ClearFieldOrExtension(message, field)665          self._ConvertMapFieldValue(666              value, message, field, '{0}.{1}'.format(path, name)667          )668        elif field.is_repeated:669          _ClearFieldOrExtension(message, field)670          if not isinstance(value, _LIST_LIKE):671            raise ParseError(672                'repeated field {0} must be in [] which is {1} at {2}'.format(673                    name, value, path674                )675            )676          if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:677            # Repeated message field.678            for index, item in enumerate(value):679              sub_message = _GetFieldOrExtension(message, field).add()680              # None is a null_value in Value.681              if (682                  item is None683                  and sub_message.DESCRIPTOR.full_name684                  != 'google.protobuf.Value'685              ):686                raise ParseError(687                    'null is not allowed to be used as an element'688                    ' in a repeated field at {0}.{1}[{2}]'.format(689                        path, name, index690                    )691                )692              self.ConvertMessage(693                  item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)694              )695          else:696            # Repeated scalar field.697            for index, item in enumerate(value):698              if item is None:699                raise ParseError(700                    'null is not allowed to be used as an element'701                    ' in a repeated field at {0}.{1}[{2}]'.format(702                        path, name, index703                    )704                )705              self._ConvertAndAppendScalar(706                  message, field, item, '{0}.{1}[{2}]'.format(path, name, index)707              )708        elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:709          sub_message = _GetFieldOrExtension(message, field)710          sub_message.SetInParent()711          self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))712        else:713          self._ConvertAndSetScalar(714              message, field, value, '{0}.{1}'.format(path, name)715          )716      except ParseError as e:717        if field and field.containing_oneof is None:718          raise ParseError(719              'Failed to parse {0} field: {1}.'.format(name, e)720          ) from e721        else:722          raise ParseError(str(e)) from e723      except ValueError as e:724        raise ParseError(725            'Failed to parse {0} field: {1}.'.format(name, e)726        ) from e727      except TypeError as e:728        raise ParseError(729            'Failed to parse {0} field: {1}.'.format(name, e)730        ) from e731 732  def _ConvertAnyMessage(self, value, message, path):733    """Convert a JSON representation into Any message."""734    if isinstance(value, dict) and not value:735      return736    try:737      type_url = value['@type']738    except KeyError as e:739      raise ParseError(740          '@type is missing when parsing any message at {0}'.format(path)741      ) from e742 743    try:744      sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)745    except TypeError as e:746      raise ParseError('{0} at {1}'.format(e, path)) from e747    message_descriptor = sub_message.DESCRIPTOR748    full_name = message_descriptor.full_name749    if _IsWrapperMessage(message_descriptor):750      self._ConvertWrapperMessage(751          value['value'], sub_message, '{0}.value'.format(path)752      )753    elif full_name in _WKTJSONMETHODS:754      # For well-known types (including nested Any), use ConvertMessage755      # to ensure recursion depth is properly tracked756      self.ConvertMessage(757          value['value'], sub_message, '{0}.value'.format(path)758      )759    else:760      del value['@type']761      try:762        self._ConvertFieldValuePair(value, sub_message, path)763      finally:764        value['@type'] = type_url765    # Sets Any message766    message.value = sub_message.SerializeToString()767    message.type_url = type_url768 769  def _ConvertGenericMessage(self, value, message, path):770    """Convert a JSON representation into message with FromJsonString."""771    # Duration, Timestamp, FieldMask have a FromJsonString method to do the772    # conversion. Users can also call the method directly.773    try:774      message.FromJsonString(value)775    except ValueError as e:776      raise ParseError('{0} at {1}'.format(e, path)) from e777 778  def _ConvertValueMessage(self, value, message, path):779    """Convert a JSON representation into Value message."""780    if isinstance(value, dict):781      self._ConvertStructMessage(value, message.struct_value, path)782    elif isinstance(value, _LIST_LIKE):783      self._ConvertListOrTupleValueMessage(value, message.list_value, path)784    elif value is None:785      message.null_value = 0786    elif isinstance(value, bool):787      message.bool_value = value788    elif isinstance(value, str):789      message.string_value = value790    elif isinstance(value, _INT_OR_FLOAT):791      message.number_value = value792    else:793      raise ParseError(794          'Value {0} has unexpected type {1} at {2}'.format(795              value, type(value), path796          )797      )798 799  def _ConvertListOrTupleValueMessage(self, value, message, path):800    """Convert a JSON representation into ListValue message."""801    if not isinstance(value, _LIST_LIKE):802      raise ParseError(803          'ListValue must be in [] which is {0} at {1}'.format(value, path)804      )805    message.ClearField('values')806    for index, item in enumerate(value):807      self._ConvertValueMessage(808          item, message.values.add(), '{0}[{1}]'.format(path, index)809      )810 811  def _ConvertStructMessage(self, value, message, path):812    """Convert a JSON representation into Struct message."""813    if not isinstance(value, dict):814      raise ParseError(815          'Struct must be in a dict which is {0} at {1}'.format(value, path)816      )817    # Clear will mark the struct as modified so it will be created even if818    # there are no values.819    message.Clear()820    for key in value:821      self._ConvertValueMessage(822          value[key], message.fields[key], '{0}.{1}'.format(path, key)823      )824    return825 826  def _ConvertWrapperMessage(self, value, message, path):827    """Convert a JSON representation into Wrapper message."""828    field = message.DESCRIPTOR.fields_by_name['value']829    self._ConvertAndSetScalar(830        message, field, value, path='{0}.value'.format(path)831    )832 833  def _ConvertMapFieldValue(self, value, message, field, path):834    """Convert map field value for a message map field.835 836    Args:837      value: A JSON object to convert the map field value.838      message: A protocol message to record the converted data.839      field: The descriptor of the map field to be converted.840      path: parent path to log parse error info.841 842    Raises:843      ParseError: In case of convert problems.844    """845    if not isinstance(value, dict):846      raise ParseError(847          'Map field {0} must be in a dict which is {1} at {2}'.format(848              field.name, value, path849          )850      )851    key_field = field.message_type.fields_by_name['key']852    value_field = field.message_type.fields_by_name['value']853    for key in value:854      key_value = _ConvertScalarFieldValue(855          key, key_field, '{0}.key'.format(path), True856      )857      if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:858        self.ConvertMessage(859            value[key],860            getattr(message, field.name)[key_value],861            '{0}[{1}]'.format(path, key_value),862        )863      else:864        self._ConvertAndSetScalarToMapKey(865            message,866            field,867            key_value,868            value[key],869            path='{0}[{1}]'.format(path, key_value),870        )871 872  def _ConvertAndSetScalar(self, message, field, js_value, path):873    """Convert scalar from js_value and assign it to message.field."""874    try:875      value = _ConvertScalarFieldValue(js_value, field, path)876      if field.is_extension:877        message.Extensions[field] = value878      else:879        setattr(message, field.name, value)880    except EnumStringValueParseError:881      if not self.ignore_unknown_fields:882        raise883 884  def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path):885    """Convert scalar from js_value and append it to message.repeated_field."""886    try:887      if repeated_field.is_extension:888        repeated = message.Extensions[repeated_field]889      else:890        repeated = getattr(message, repeated_field.name)891      value = _ConvertScalarFieldValue(js_value, repeated_field, path)892      repeated.append(value)893    except EnumStringValueParseError:894      if not self.ignore_unknown_fields:895        raise896 897  def _ConvertAndSetScalarToMapKey(898      self, message, map_field, converted_key, js_value, path899  ):900    """Convert scalar from 'js_value' and add it to message.map_field[converted_key]."""901    try:902      getattr(message, map_field.name)[converted_key] = (903          _ConvertScalarFieldValue(904              js_value,905              map_field.message_type.fields_by_name['value'],906              path,907          )908      )909    except EnumStringValueParseError:910      if not self.ignore_unknown_fields:911        raise912 913 914def _ConvertScalarFieldValue(value, field, path, require_str=False):915  """Convert a single scalar field value.916 917  Args:918    value: A scalar value to convert the scalar field value.919    field: The descriptor of the field to convert.920    path: parent path to log parse error info.921    require_str: If True, the field value must be a str.922 923  Returns:924    The converted scalar field value925 926  Raises:927    ParseError: In case of convert problems.928    EnumStringValueParseError: In case of unknown enum string value.929  """930  try:931    if field.cpp_type in _INT_TYPES:932      return _ConvertInteger(value)933    elif field.cpp_type in _FLOAT_TYPES:934      return _ConvertFloat(value, field)935    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:936      return _ConvertBool(value, require_str)937    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:938      if field.type == descriptor.FieldDescriptor.TYPE_BYTES:939        if isinstance(value, str):940          encoded = value.encode('utf-8')941        else:942          encoded = value943        # Add extra padding '='944        padded_value = encoded + b'=' * (4 - len(encoded) % 4)945        return base64.urlsafe_b64decode(padded_value)946      else:947        # Checking for unpaired surrogates appears to be unreliable,948        # depending on the specific Python version, so we check manually.949        if _UNPAIRED_SURROGATE_PATTERN.search(value):950          raise ParseError('Unpaired surrogate')951        return value952    elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:953      # Convert an enum value.954      enum_value = field.enum_type.values_by_name.get(value, None)955      if enum_value is None:956        try:957          number = int(value)958          enum_value = field.enum_type.values_by_number.get(number, None)959        except ValueError as e:960          # Since parsing to integer failed and lookup in values_by_name didn't961          # find this name, we have an enum string value which is unknown.962          raise EnumStringValueParseError(963              'Invalid enum value {0} for enum type {1}'.format(964                  value, field.enum_type.full_name965              )966          ) from e967        if enum_value is None:968          if field.enum_type.is_closed:969            raise ParseError(970                'Invalid enum value {0} for enum type {1}'.format(971                    value, field.enum_type.full_name972                )973            )974          else:975            return number976      return enum_value.number977  except EnumStringValueParseError as e:978    raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e979  except ParseError as e:980    raise ParseError('{0} at {1}'.format(e, path)) from e981 982 983def _ConvertInteger(value):984  """Convert an integer.985 986  Args:987    value: A scalar value to convert.988 989  Returns:990    The integer value.991 992  Raises:993    ParseError: If an integer couldn't be consumed.994  """995  if isinstance(value, float) and not value.is_integer():996    raise ParseError("Couldn't parse integer: {0}".format(value))997 998  if isinstance(value, str) and value.find(' ') != -1:999    raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))1000 1001  if isinstance(value, bool):1002    raise ParseError(1003        'Bool value {0} is not acceptable for integer field'.format(value)1004    )1005 1006  try:1007    return int(value)1008  except ValueError as e:1009    # Attempt to parse as an integer-valued float.1010    try:1011      f = float(value)1012    except ValueError:1013      # Raise the original exception for the int parse.1014      raise e  # pylint: disable=raise-missing-from1015    if not f.is_integer():1016      raise ParseError(1017          'Couldn\'t parse non-integer string: "{0}"'.format(value)1018      ) from e1019    return int(f)1020 1021 1022def _ConvertFloat(value, field):1023  """Convert an floating point number."""1024  if isinstance(value, float):1025    if math.isnan(value):1026      raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')1027    if math.isinf(value):1028      if value > 0:1029        raise ParseError(1030            "Couldn't parse Infinity or value too large, "1031            'use quoted "Infinity" instead'1032        )1033      else:1034        raise ParseError(1035            "Couldn't parse -Infinity or value too small, "1036            'use quoted "-Infinity" instead'1037        )1038    if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:1039      # pylint: disable=protected-access1040      if value > type_checkers._FLOAT_MAX:1041        raise ParseError('Float value too large')1042      # pylint: disable=protected-access1043      if value < type_checkers._FLOAT_MIN:1044        raise ParseError('Float value too small')1045  if value == 'nan':1046    raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')1047  try:1048    # Assume Python compatible syntax.1049    return float(value)1050  except ValueError as e:1051    # Check alternative spellings.1052    if value == _NEG_INFINITY:1053      return float('-inf')1054    elif value == _INFINITY:1055      return float('inf')1056    elif value == _NAN:1057      return float('nan')1058    else:1059      raise ParseError("Couldn't parse float: {0}".format(value)) from e1060 1061 1062def _ConvertBool(value, require_str):1063  """Convert a boolean value.1064 1065  Args:1066    value: A scalar value to convert.1067    require_str: If True, value must be a str.1068 1069  Returns:1070    The bool parsed.1071 1072  Raises:1073    ParseError: If a boolean value couldn't be consumed.1074  """1075  if require_str:1076    if value == 'true':1077      return True1078    elif value == 'false':1079      return False1080    else:1081      raise ParseError('Expected "true" or "false", not {0}'.format(value))1082 1083  if not isinstance(value, bool):1084    raise ParseError('Expected true or false without quotes')1085  return value1086 1087 1088_WKTJSONMETHODS = {1089    'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],1090    'google.protobuf.Duration': [1091        '_GenericMessageToJsonObject',1092        '_ConvertGenericMessage',1093    ],1094    'google.protobuf.FieldMask': [1095        '_GenericMessageToJsonObject',1096        '_ConvertGenericMessage',1097    ],1098    'google.protobuf.ListValue': [1099        '_ListValueMessageToJsonObject',1100        '_ConvertListOrTupleValueMessage',1101    ],1102    'google.protobuf.Struct': [1103        '_StructMessageToJsonObject',1104        '_ConvertStructMessage',1105    ],1106    'google.protobuf.Timestamp': [1107        '_GenericMessageToJsonObject',1108        '_ConvertGenericMessage',1109    ],1110    'google.protobuf.Value': [1111        '_ValueMessageToJsonObject',1112        '_ConvertValueMessage',1113    ],1114}1115