图片解析应用
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

929 lines
35 KiB

  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. # https://developers.google.com/protocol-buffers/
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """Contains routines for printing protocol messages in JSON format.
  31. Simple usage example:
  32. # Create a proto object and serialize it to a json format string.
  33. message = my_proto_pb2.MyMessage(foo='bar')
  34. json_string = json_format.MessageToJson(message)
  35. # Parse a json format string to proto object.
  36. message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
  37. """
  38. __author__ = 'jieluo@google.com (Jie Luo)'
  39. import base64
  40. from collections import OrderedDict
  41. import json
  42. import math
  43. from operator import methodcaller
  44. import re
  45. import sys
  46. from google.protobuf.internal import type_checkers
  47. from google.protobuf import descriptor
  48. from google.protobuf import message_factory
  49. from google.protobuf import symbol_database
  50. _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S'
  51. _INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32,
  52. descriptor.FieldDescriptor.CPPTYPE_UINT32,
  53. descriptor.FieldDescriptor.CPPTYPE_INT64,
  54. descriptor.FieldDescriptor.CPPTYPE_UINT64])
  55. _INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64,
  56. descriptor.FieldDescriptor.CPPTYPE_UINT64])
  57. _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
  58. descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
  59. _INFINITY = 'Infinity'
  60. _NEG_INFINITY = '-Infinity'
  61. _NAN = 'NaN'
  62. _UNPAIRED_SURROGATE_PATTERN = re.compile(
  63. u'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]')
  64. _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
  65. class Error(Exception):
  66. """Top-level module error for json_format."""
  67. class SerializeToJsonError(Error):
  68. """Thrown if serialization to JSON fails."""
  69. class ParseError(Error):
  70. """Thrown in case of parsing error."""
  71. def MessageToJson(
  72. message,
  73. including_default_value_fields=False,
  74. preserving_proto_field_name=False,
  75. indent=2,
  76. sort_keys=False,
  77. use_integers_for_enums=False,
  78. descriptor_pool=None,
  79. float_precision=None,
  80. ensure_ascii=True):
  81. """Converts protobuf message to JSON format.
  82. Args:
  83. message: The protocol buffers message instance to serialize.
  84. including_default_value_fields: If True, singular primitive fields,
  85. repeated fields, and map fields will always be serialized. If
  86. False, only serialize non-empty fields. Singular message fields
  87. and oneof fields are not affected by this option.
  88. preserving_proto_field_name: If True, use the original proto field
  89. names as defined in the .proto file. If False, convert the field
  90. names to lowerCamelCase.
  91. indent: The JSON object will be pretty-printed with this indent level.
  92. An indent level of 0 or negative will only insert newlines. If the
  93. indent level is None, no newlines will be inserted.
  94. sort_keys: If True, then the output will be sorted by field names.
  95. use_integers_for_enums: If true, print integers instead of enum names.
  96. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  97. default.
  98. float_precision: If set, use this to specify float field valid digits.
  99. ensure_ascii: If True, strings with non-ASCII characters are escaped.
  100. If False, Unicode strings are returned unchanged.
  101. Returns:
  102. A string containing the JSON formatted protocol buffer message.
  103. """
  104. printer = _Printer(
  105. including_default_value_fields,
  106. preserving_proto_field_name,
  107. use_integers_for_enums,
  108. descriptor_pool,
  109. float_precision=float_precision)
  110. return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
  111. def MessageToDict(
  112. message,
  113. including_default_value_fields=False,
  114. preserving_proto_field_name=False,
  115. use_integers_for_enums=False,
  116. descriptor_pool=None,
  117. float_precision=None):
  118. """Converts protobuf message to a dictionary.
  119. When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
  120. Args:
  121. message: The protocol buffers message instance to serialize.
  122. including_default_value_fields: If True, singular primitive fields,
  123. repeated fields, and map fields will always be serialized. If
  124. False, only serialize non-empty fields. Singular message fields
  125. and oneof fields are not affected by this option.
  126. preserving_proto_field_name: If True, use the original proto field
  127. names as defined in the .proto file. If False, convert the field
  128. names to lowerCamelCase.
  129. use_integers_for_enums: If true, print integers instead of enum names.
  130. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  131. default.
  132. float_precision: If set, use this to specify float field valid digits.
  133. Returns:
  134. A dict representation of the protocol buffer message.
  135. """
  136. printer = _Printer(
  137. including_default_value_fields,
  138. preserving_proto_field_name,
  139. use_integers_for_enums,
  140. descriptor_pool,
  141. float_precision=float_precision)
  142. # pylint: disable=protected-access
  143. return printer._MessageToJsonObject(message)
  144. def _IsMapEntry(field):
  145. return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
  146. field.message_type.has_options and
  147. field.message_type.GetOptions().map_entry)
  148. class _Printer(object):
  149. """JSON format printer for protocol message."""
  150. def __init__(
  151. self,
  152. including_default_value_fields=False,
  153. preserving_proto_field_name=False,
  154. use_integers_for_enums=False,
  155. descriptor_pool=None,
  156. float_precision=None):
  157. self.including_default_value_fields = including_default_value_fields
  158. self.preserving_proto_field_name = preserving_proto_field_name
  159. self.use_integers_for_enums = use_integers_for_enums
  160. self.descriptor_pool = descriptor_pool
  161. if float_precision:
  162. self.float_format = '.{}g'.format(float_precision)
  163. else:
  164. self.float_format = None
  165. def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
  166. js = self._MessageToJsonObject(message)
  167. return json.dumps(
  168. js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii)
  169. def _MessageToJsonObject(self, message):
  170. """Converts message to an object according to Proto3 JSON Specification."""
  171. message_descriptor = message.DESCRIPTOR
  172. full_name = message_descriptor.full_name
  173. if _IsWrapperMessage(message_descriptor):
  174. return self._WrapperMessageToJsonObject(message)
  175. if full_name in _WKTJSONMETHODS:
  176. return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)
  177. js = {}
  178. return self._RegularMessageToJsonObject(message, js)
  179. def _RegularMessageToJsonObject(self, message, js):
  180. """Converts normal message according to Proto3 JSON Specification."""
  181. fields = message.ListFields()
  182. try:
  183. for field, value in fields:
  184. if self.preserving_proto_field_name:
  185. name = field.name
  186. else:
  187. name = field.json_name
  188. if _IsMapEntry(field):
  189. # Convert a map field.
  190. v_field = field.message_type.fields_by_name['value']
  191. js_map = {}
  192. for key in value:
  193. if isinstance(key, bool):
  194. if key:
  195. recorded_key = 'true'
  196. else:
  197. recorded_key = 'false'
  198. else:
  199. recorded_key = str(key)
  200. js_map[recorded_key] = self._FieldToJsonObject(
  201. v_field, value[key])
  202. js[name] = js_map
  203. elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  204. # Convert a repeated field.
  205. js[name] = [self._FieldToJsonObject(field, k)
  206. for k in value]
  207. elif field.is_extension:
  208. name = '[%s]' % field.full_name
  209. js[name] = self._FieldToJsonObject(field, value)
  210. else:
  211. js[name] = self._FieldToJsonObject(field, value)
  212. # Serialize default value if including_default_value_fields is True.
  213. if self.including_default_value_fields:
  214. message_descriptor = message.DESCRIPTOR
  215. for field in message_descriptor.fields:
  216. # Singular message fields and oneof fields will not be affected.
  217. if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and
  218. field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or
  219. field.containing_oneof):
  220. continue
  221. if self.preserving_proto_field_name:
  222. name = field.name
  223. else:
  224. name = field.json_name
  225. if name in js:
  226. # Skip the field which has been serialized already.
  227. continue
  228. if _IsMapEntry(field):
  229. js[name] = {}
  230. elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  231. js[name] = []
  232. else:
  233. js[name] = self._FieldToJsonObject(field, field.default_value)
  234. except ValueError as e:
  235. raise SerializeToJsonError(
  236. 'Failed to serialize {0} field: {1}.'.format(field.name, e)) from e
  237. return js
  238. def _FieldToJsonObject(self, field, value):
  239. """Converts field value according to Proto3 JSON Specification."""
  240. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  241. return self._MessageToJsonObject(value)
  242. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  243. if self.use_integers_for_enums:
  244. return value
  245. if field.enum_type.full_name == 'google.protobuf.NullValue':
  246. return None
  247. enum_value = field.enum_type.values_by_number.get(value, None)
  248. if enum_value is not None:
  249. return enum_value.name
  250. else:
  251. if field.enum_type.is_closed:
  252. raise SerializeToJsonError('Enum field contains an integer value '
  253. 'which can not mapped to an enum value.')
  254. else:
  255. return value
  256. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  257. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  258. # Use base64 Data encoding for bytes
  259. return base64.b64encode(value).decode('utf-8')
  260. else:
  261. return value
  262. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  263. return bool(value)
  264. elif field.cpp_type in _INT64_TYPES:
  265. return str(value)
  266. elif field.cpp_type in _FLOAT_TYPES:
  267. if math.isinf(value):
  268. if value < 0.0:
  269. return _NEG_INFINITY
  270. else:
  271. return _INFINITY
  272. if math.isnan(value):
  273. return _NAN
  274. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  275. if self.float_format:
  276. return float(format(value, self.float_format))
  277. else:
  278. return type_checkers.ToShortestFloat(value)
  279. return value
  280. def _AnyMessageToJsonObject(self, message):
  281. """Converts Any message according to Proto3 JSON Specification."""
  282. if not message.ListFields():
  283. return {}
  284. # Must print @type first, use OrderedDict instead of {}
  285. js = OrderedDict()
  286. type_url = message.type_url
  287. js['@type'] = type_url
  288. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  289. sub_message.ParseFromString(message.value)
  290. message_descriptor = sub_message.DESCRIPTOR
  291. full_name = message_descriptor.full_name
  292. if _IsWrapperMessage(message_descriptor):
  293. js['value'] = self._WrapperMessageToJsonObject(sub_message)
  294. return js
  295. if full_name in _WKTJSONMETHODS:
  296. js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0],
  297. sub_message)(self)
  298. return js
  299. return self._RegularMessageToJsonObject(sub_message, js)
  300. def _GenericMessageToJsonObject(self, message):
  301. """Converts message according to Proto3 JSON Specification."""
  302. # Duration, Timestamp and FieldMask have ToJsonString method to do the
  303. # convert. Users can also call the method directly.
  304. return message.ToJsonString()
  305. def _ValueMessageToJsonObject(self, message):
  306. """Converts Value message according to Proto3 JSON Specification."""
  307. which = message.WhichOneof('kind')
  308. # If the Value message is not set treat as null_value when serialize
  309. # to JSON. The parse back result will be different from original message.
  310. if which is None or which == 'null_value':
  311. return None
  312. if which == 'list_value':
  313. return self._ListValueMessageToJsonObject(message.list_value)
  314. if which == 'number_value':
  315. value = message.number_value
  316. if math.isinf(value):
  317. raise ValueError('Fail to serialize Infinity for Value.number_value, '
  318. 'which would parse as string_value')
  319. if math.isnan(value):
  320. raise ValueError('Fail to serialize NaN for Value.number_value, '
  321. 'which would parse as string_value')
  322. else:
  323. value = getattr(message, which)
  324. oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
  325. return self._FieldToJsonObject(oneof_descriptor, value)
  326. def _ListValueMessageToJsonObject(self, message):
  327. """Converts ListValue message according to Proto3 JSON Specification."""
  328. return [self._ValueMessageToJsonObject(value)
  329. for value in message.values]
  330. def _StructMessageToJsonObject(self, message):
  331. """Converts Struct message according to Proto3 JSON Specification."""
  332. fields = message.fields
  333. ret = {}
  334. for key in fields:
  335. ret[key] = self._ValueMessageToJsonObject(fields[key])
  336. return ret
  337. def _WrapperMessageToJsonObject(self, message):
  338. return self._FieldToJsonObject(
  339. message.DESCRIPTOR.fields_by_name['value'], message.value)
  340. def _IsWrapperMessage(message_descriptor):
  341. return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
  342. def _DuplicateChecker(js):
  343. result = {}
  344. for name, value in js:
  345. if name in result:
  346. raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
  347. result[name] = value
  348. return result
  349. def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
  350. """Creates a message from a type URL."""
  351. db = symbol_database.Default()
  352. pool = db.pool if descriptor_pool is None else descriptor_pool
  353. type_name = type_url.split('/')[-1]
  354. try:
  355. message_descriptor = pool.FindMessageTypeByName(type_name)
  356. except KeyError as e:
  357. raise TypeError(
  358. 'Can not find message descriptor by type_url: {0}'.format(type_url)
  359. ) from e
  360. message_class = message_factory.GetMessageClass(message_descriptor)
  361. return message_class()
  362. def Parse(text,
  363. message,
  364. ignore_unknown_fields=False,
  365. descriptor_pool=None,
  366. max_recursion_depth=100):
  367. """Parses a JSON representation of a protocol message into a message.
  368. Args:
  369. text: Message JSON representation.
  370. message: A protocol buffer message to merge into.
  371. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  372. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  373. default.
  374. max_recursion_depth: max recursion depth of JSON message to be
  375. deserialized. JSON messages over this depth will fail to be
  376. deserialized. Default value is 100.
  377. Returns:
  378. The same message passed as argument.
  379. Raises::
  380. ParseError: On JSON parsing problems.
  381. """
  382. if not isinstance(text, str):
  383. text = text.decode('utf-8')
  384. try:
  385. js = json.loads(text, object_pairs_hook=_DuplicateChecker)
  386. except ValueError as e:
  387. raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
  388. return ParseDict(js, message, ignore_unknown_fields, descriptor_pool,
  389. max_recursion_depth)
  390. def ParseDict(js_dict,
  391. message,
  392. ignore_unknown_fields=False,
  393. descriptor_pool=None,
  394. max_recursion_depth=100):
  395. """Parses a JSON dictionary representation into a message.
  396. Args:
  397. js_dict: Dict representation of a JSON message.
  398. message: A protocol buffer message to merge into.
  399. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  400. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  401. default.
  402. max_recursion_depth: max recursion depth of JSON message to be
  403. deserialized. JSON messages over this depth will fail to be
  404. deserialized. Default value is 100.
  405. Returns:
  406. The same message passed as argument.
  407. """
  408. parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
  409. parser.ConvertMessage(js_dict, message, '')
  410. return message
  411. _INT_OR_FLOAT = (int, float)
  412. class _Parser(object):
  413. """JSON format parser for protocol message."""
  414. def __init__(self, ignore_unknown_fields, descriptor_pool,
  415. max_recursion_depth):
  416. self.ignore_unknown_fields = ignore_unknown_fields
  417. self.descriptor_pool = descriptor_pool
  418. self.max_recursion_depth = max_recursion_depth
  419. self.recursion_depth = 0
  420. def ConvertMessage(self, value, message, path):
  421. """Convert a JSON object into a message.
  422. Args:
  423. value: A JSON object.
  424. message: A WKT or regular protocol message to record the data.
  425. path: parent path to log parse error info.
  426. Raises:
  427. ParseError: In case of convert problems.
  428. """
  429. self.recursion_depth += 1
  430. if self.recursion_depth > self.max_recursion_depth:
  431. raise ParseError('Message too deep. Max recursion depth is {0}'.format(
  432. self.max_recursion_depth))
  433. message_descriptor = message.DESCRIPTOR
  434. full_name = message_descriptor.full_name
  435. if not path:
  436. path = message_descriptor.name
  437. if _IsWrapperMessage(message_descriptor):
  438. self._ConvertWrapperMessage(value, message, path)
  439. elif full_name in _WKTJSONMETHODS:
  440. methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
  441. else:
  442. self._ConvertFieldValuePair(value, message, path)
  443. self.recursion_depth -= 1
  444. def _ConvertFieldValuePair(self, js, message, path):
  445. """Convert field value pairs into regular message.
  446. Args:
  447. js: A JSON object to convert the field value pairs.
  448. message: A regular protocol message to record the data.
  449. path: parent path to log parse error info.
  450. Raises:
  451. ParseError: In case of problems converting.
  452. """
  453. names = []
  454. message_descriptor = message.DESCRIPTOR
  455. fields_by_json_name = dict((f.json_name, f)
  456. for f in message_descriptor.fields)
  457. for name in js:
  458. try:
  459. field = fields_by_json_name.get(name, None)
  460. if not field:
  461. field = message_descriptor.fields_by_name.get(name, None)
  462. if not field and _VALID_EXTENSION_NAME.match(name):
  463. if not message_descriptor.is_extendable:
  464. raise ParseError(
  465. 'Message type {0} does not have extensions at {1}'.format(
  466. message_descriptor.full_name, path))
  467. identifier = name[1:-1] # strip [] brackets
  468. # pylint: disable=protected-access
  469. field = message.Extensions._FindExtensionByName(identifier)
  470. # pylint: enable=protected-access
  471. if not field:
  472. # Try looking for extension by the message type name, dropping the
  473. # field name following the final . separator in full_name.
  474. identifier = '.'.join(identifier.split('.')[:-1])
  475. # pylint: disable=protected-access
  476. field = message.Extensions._FindExtensionByName(identifier)
  477. # pylint: enable=protected-access
  478. if not field:
  479. if self.ignore_unknown_fields:
  480. continue
  481. raise ParseError(
  482. ('Message type "{0}" has no field named "{1}" at "{2}".\n'
  483. ' Available Fields(except extensions): "{3}"').format(
  484. message_descriptor.full_name, name, path,
  485. [f.json_name for f in message_descriptor.fields]))
  486. if name in names:
  487. raise ParseError('Message type "{0}" should not have multiple '
  488. '"{1}" fields at "{2}".'.format(
  489. message.DESCRIPTOR.full_name, name, path))
  490. names.append(name)
  491. value = js[name]
  492. # Check no other oneof field is parsed.
  493. if field.containing_oneof is not None and value is not None:
  494. oneof_name = field.containing_oneof.name
  495. if oneof_name in names:
  496. raise ParseError('Message type "{0}" should not have multiple '
  497. '"{1}" oneof fields at "{2}".'.format(
  498. message.DESCRIPTOR.full_name, oneof_name,
  499. path))
  500. names.append(oneof_name)
  501. if value is None:
  502. if (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
  503. and field.message_type.full_name == 'google.protobuf.Value'):
  504. sub_message = getattr(message, field.name)
  505. sub_message.null_value = 0
  506. elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
  507. and field.enum_type.full_name == 'google.protobuf.NullValue'):
  508. setattr(message, field.name, 0)
  509. else:
  510. message.ClearField(field.name)
  511. continue
  512. # Parse field value.
  513. if _IsMapEntry(field):
  514. message.ClearField(field.name)
  515. self._ConvertMapFieldValue(value, message, field,
  516. '{0}.{1}'.format(path, name))
  517. elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  518. message.ClearField(field.name)
  519. if not isinstance(value, list):
  520. raise ParseError('repeated field {0} must be in [] which is '
  521. '{1} at {2}'.format(name, value, path))
  522. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  523. # Repeated message field.
  524. for index, item in enumerate(value):
  525. sub_message = getattr(message, field.name).add()
  526. # None is a null_value in Value.
  527. if (item is None and
  528. sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'):
  529. raise ParseError('null is not allowed to be used as an element'
  530. ' in a repeated field at {0}.{1}[{2}]'.format(
  531. path, name, index))
  532. self.ConvertMessage(item, sub_message,
  533. '{0}.{1}[{2}]'.format(path, name, index))
  534. else:
  535. # Repeated scalar field.
  536. for index, item in enumerate(value):
  537. if item is None:
  538. raise ParseError('null is not allowed to be used as an element'
  539. ' in a repeated field at {0}.{1}[{2}]'.format(
  540. path, name, index))
  541. getattr(message, field.name).append(
  542. _ConvertScalarFieldValue(
  543. item, field, '{0}.{1}[{2}]'.format(path, name, index)))
  544. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  545. if field.is_extension:
  546. sub_message = message.Extensions[field]
  547. else:
  548. sub_message = getattr(message, field.name)
  549. sub_message.SetInParent()
  550. self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
  551. else:
  552. if field.is_extension:
  553. message.Extensions[field] = _ConvertScalarFieldValue(
  554. value, field, '{0}.{1}'.format(path, name))
  555. else:
  556. setattr(
  557. message, field.name,
  558. _ConvertScalarFieldValue(value, field,
  559. '{0}.{1}'.format(path, name)))
  560. except ParseError as e:
  561. if field and field.containing_oneof is None:
  562. raise ParseError(
  563. 'Failed to parse {0} field: {1}.'.format(name, e)
  564. ) from e
  565. else:
  566. raise ParseError(str(e)) from e
  567. except ValueError as e:
  568. raise ParseError(
  569. 'Failed to parse {0} field: {1}.'.format(name, e)
  570. ) from e
  571. except TypeError as e:
  572. raise ParseError(
  573. 'Failed to parse {0} field: {1}.'.format(name, e)
  574. ) from e
  575. def _ConvertAnyMessage(self, value, message, path):
  576. """Convert a JSON representation into Any message."""
  577. if isinstance(value, dict) and not value:
  578. return
  579. try:
  580. type_url = value['@type']
  581. except KeyError as e:
  582. raise ParseError(
  583. '@type is missing when parsing any message at {0}'.format(path)
  584. ) from e
  585. try:
  586. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  587. except TypeError as e:
  588. raise ParseError('{0} at {1}'.format(e, path)) from e
  589. message_descriptor = sub_message.DESCRIPTOR
  590. full_name = message_descriptor.full_name
  591. if _IsWrapperMessage(message_descriptor):
  592. self._ConvertWrapperMessage(value['value'], sub_message,
  593. '{0}.value'.format(path))
  594. elif full_name in _WKTJSONMETHODS:
  595. methodcaller(_WKTJSONMETHODS[full_name][1], value['value'], sub_message,
  596. '{0}.value'.format(path))(
  597. self)
  598. else:
  599. del value['@type']
  600. self._ConvertFieldValuePair(value, sub_message, path)
  601. value['@type'] = type_url
  602. # Sets Any message
  603. message.value = sub_message.SerializeToString()
  604. message.type_url = type_url
  605. def _ConvertGenericMessage(self, value, message, path):
  606. """Convert a JSON representation into message with FromJsonString."""
  607. # Duration, Timestamp, FieldMask have a FromJsonString method to do the
  608. # conversion. Users can also call the method directly.
  609. try:
  610. message.FromJsonString(value)
  611. except ValueError as e:
  612. raise ParseError('{0} at {1}'.format(e, path)) from e
  613. def _ConvertValueMessage(self, value, message, path):
  614. """Convert a JSON representation into Value message."""
  615. if isinstance(value, dict):
  616. self._ConvertStructMessage(value, message.struct_value, path)
  617. elif isinstance(value, list):
  618. self._ConvertListValueMessage(value, message.list_value, path)
  619. elif value is None:
  620. message.null_value = 0
  621. elif isinstance(value, bool):
  622. message.bool_value = value
  623. elif isinstance(value, str):
  624. message.string_value = value
  625. elif isinstance(value, _INT_OR_FLOAT):
  626. message.number_value = value
  627. else:
  628. raise ParseError('Value {0} has unexpected type {1} at {2}'.format(
  629. value, type(value), path))
  630. def _ConvertListValueMessage(self, value, message, path):
  631. """Convert a JSON representation into ListValue message."""
  632. if not isinstance(value, list):
  633. raise ParseError('ListValue must be in [] which is {0} at {1}'.format(
  634. value, path))
  635. message.ClearField('values')
  636. for index, item in enumerate(value):
  637. self._ConvertValueMessage(item, message.values.add(),
  638. '{0}[{1}]'.format(path, index))
  639. def _ConvertStructMessage(self, value, message, path):
  640. """Convert a JSON representation into Struct message."""
  641. if not isinstance(value, dict):
  642. raise ParseError('Struct must be in a dict which is {0} at {1}'.format(
  643. value, path))
  644. # Clear will mark the struct as modified so it will be created even if
  645. # there are no values.
  646. message.Clear()
  647. for key in value:
  648. self._ConvertValueMessage(value[key], message.fields[key],
  649. '{0}.{1}'.format(path, key))
  650. return
  651. def _ConvertWrapperMessage(self, value, message, path):
  652. """Convert a JSON representation into Wrapper message."""
  653. field = message.DESCRIPTOR.fields_by_name['value']
  654. setattr(
  655. message, 'value',
  656. _ConvertScalarFieldValue(value, field, path='{0}.value'.format(path)))
  657. def _ConvertMapFieldValue(self, value, message, field, path):
  658. """Convert map field value for a message map field.
  659. Args:
  660. value: A JSON object to convert the map field value.
  661. message: A protocol message to record the converted data.
  662. field: The descriptor of the map field to be converted.
  663. path: parent path to log parse error info.
  664. Raises:
  665. ParseError: In case of convert problems.
  666. """
  667. if not isinstance(value, dict):
  668. raise ParseError(
  669. 'Map field {0} must be in a dict which is {1} at {2}'.format(
  670. field.name, value, path))
  671. key_field = field.message_type.fields_by_name['key']
  672. value_field = field.message_type.fields_by_name['value']
  673. for key in value:
  674. key_value = _ConvertScalarFieldValue(key, key_field,
  675. '{0}.key'.format(path), True)
  676. if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  677. self.ConvertMessage(value[key],
  678. getattr(message, field.name)[key_value],
  679. '{0}[{1}]'.format(path, key_value))
  680. else:
  681. getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
  682. value[key], value_field, path='{0}[{1}]'.format(path, key_value))
  683. def _ConvertScalarFieldValue(value, field, path, require_str=False):
  684. """Convert a single scalar field value.
  685. Args:
  686. value: A scalar value to convert the scalar field value.
  687. field: The descriptor of the field to convert.
  688. path: parent path to log parse error info.
  689. require_str: If True, the field value must be a str.
  690. Returns:
  691. The converted scalar field value
  692. Raises:
  693. ParseError: In case of convert problems.
  694. """
  695. try:
  696. if field.cpp_type in _INT_TYPES:
  697. return _ConvertInteger(value)
  698. elif field.cpp_type in _FLOAT_TYPES:
  699. return _ConvertFloat(value, field)
  700. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  701. return _ConvertBool(value, require_str)
  702. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  703. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  704. if isinstance(value, str):
  705. encoded = value.encode('utf-8')
  706. else:
  707. encoded = value
  708. # Add extra padding '='
  709. padded_value = encoded + b'=' * (4 - len(encoded) % 4)
  710. return base64.urlsafe_b64decode(padded_value)
  711. else:
  712. # Checking for unpaired surrogates appears to be unreliable,
  713. # depending on the specific Python version, so we check manually.
  714. if _UNPAIRED_SURROGATE_PATTERN.search(value):
  715. raise ParseError('Unpaired surrogate')
  716. return value
  717. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  718. # Convert an enum value.
  719. enum_value = field.enum_type.values_by_name.get(value, None)
  720. if enum_value is None:
  721. try:
  722. number = int(value)
  723. enum_value = field.enum_type.values_by_number.get(number, None)
  724. except ValueError as e:
  725. raise ParseError('Invalid enum value {0} for enum type {1}'.format(
  726. value, field.enum_type.full_name)) from e
  727. if enum_value is None:
  728. if field.enum_type.is_closed:
  729. raise ParseError('Invalid enum value {0} for enum type {1}'.format(
  730. value, field.enum_type.full_name))
  731. else:
  732. return number
  733. return enum_value.number
  734. except ParseError as e:
  735. raise ParseError('{0} at {1}'.format(e, path)) from e
  736. def _ConvertInteger(value):
  737. """Convert an integer.
  738. Args:
  739. value: A scalar value to convert.
  740. Returns:
  741. The integer value.
  742. Raises:
  743. ParseError: If an integer couldn't be consumed.
  744. """
  745. if isinstance(value, float) and not value.is_integer():
  746. raise ParseError('Couldn\'t parse integer: {0}'.format(value))
  747. if isinstance(value, str) and value.find(' ') != -1:
  748. raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
  749. if isinstance(value, bool):
  750. raise ParseError('Bool value {0} is not acceptable for '
  751. 'integer field'.format(value))
  752. return int(value)
  753. def _ConvertFloat(value, field):
  754. """Convert an floating point number."""
  755. if isinstance(value, float):
  756. if math.isnan(value):
  757. raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
  758. if math.isinf(value):
  759. if value > 0:
  760. raise ParseError('Couldn\'t parse Infinity or value too large, '
  761. 'use quoted "Infinity" instead')
  762. else:
  763. raise ParseError('Couldn\'t parse -Infinity or value too small, '
  764. 'use quoted "-Infinity" instead')
  765. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  766. # pylint: disable=protected-access
  767. if value > type_checkers._FLOAT_MAX:
  768. raise ParseError('Float value too large')
  769. # pylint: disable=protected-access
  770. if value < type_checkers._FLOAT_MIN:
  771. raise ParseError('Float value too small')
  772. if value == 'nan':
  773. raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
  774. try:
  775. # Assume Python compatible syntax.
  776. return float(value)
  777. except ValueError as e:
  778. # Check alternative spellings.
  779. if value == _NEG_INFINITY:
  780. return float('-inf')
  781. elif value == _INFINITY:
  782. return float('inf')
  783. elif value == _NAN:
  784. return float('nan')
  785. else:
  786. raise ParseError('Couldn\'t parse float: {0}'.format(value)) from e
  787. def _ConvertBool(value, require_str):
  788. """Convert a boolean value.
  789. Args:
  790. value: A scalar value to convert.
  791. require_str: If True, value must be a str.
  792. Returns:
  793. The bool parsed.
  794. Raises:
  795. ParseError: If a boolean value couldn't be consumed.
  796. """
  797. if require_str:
  798. if value == 'true':
  799. return True
  800. elif value == 'false':
  801. return False
  802. else:
  803. raise ParseError('Expected "true" or "false", not {0}'.format(value))
  804. if not isinstance(value, bool):
  805. raise ParseError('Expected true or false without quotes')
  806. return value
  807. _WKTJSONMETHODS = {
  808. 'google.protobuf.Any': ['_AnyMessageToJsonObject',
  809. '_ConvertAnyMessage'],
  810. 'google.protobuf.Duration': ['_GenericMessageToJsonObject',
  811. '_ConvertGenericMessage'],
  812. 'google.protobuf.FieldMask': ['_GenericMessageToJsonObject',
  813. '_ConvertGenericMessage'],
  814. 'google.protobuf.ListValue': ['_ListValueMessageToJsonObject',
  815. '_ConvertListValueMessage'],
  816. 'google.protobuf.Struct': ['_StructMessageToJsonObject',
  817. '_ConvertStructMessage'],
  818. 'google.protobuf.Timestamp': ['_GenericMessageToJsonObject',
  819. '_ConvertGenericMessage'],
  820. 'google.protobuf.Value': ['_ValueMessageToJsonObject',
  821. '_ConvertValueMessage']
  822. }