Python google.protobuf.text_format.Parse() Examples

def testFailsOnFixedFeature(self):
    component_spec = spec_pb2.ComponentSpec()
    text_format.Parse("""
        name: "annotate"
        network_unit {
          registered_name: "IdentityNetwork"
        }
        fixed_feature {
          name: "fixed" embedding_dim: 32 size: 1
        }
        """, component_spec)
    with tf.Graph().as_default():
      comp = bulk_component.BulkAnnotatorComponentBuilder(
          self.master, component_spec)

      # Expect feature extraction to generate a runtime error due to the
      # fixed feature.
      with self.assertRaises(RuntimeError):
        comp.build_greedy_training(self.master_state, self.network_states) 

def testParseMessageSet(self):
    message = unittest_pb2.TestAllTypes()
    text = ('repeated_uint64: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('message_set {\n'
            '  [protobuf_unittest.TestMessageSetExtension1] {\n'
            '    i: 23\n'
            '  }\n'
            '  [protobuf_unittest.TestMessageSetExtension2] {\n'
            '    str: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str) 

def testMergeExpandedAny(self):
    message = any_test_pb2.TestAny()
    text = ('any_value {\n'
            '  [type.googleapis.com/protobuf_unittest.OneString] {\n'
            '    data: "string"\n'
            '  }\n'
            '}\n')
    text_format.Merge(text, message)
    packed_message = unittest_pb2.OneString()
    message.any_value.Unpack(packed_message)
    self.assertEqual('string', packed_message.data)
    message.Clear()
    text_format.Parse(text, message)
    packed_message = unittest_pb2.OneString()
    message.any_value.Unpack(packed_message)
    self.assertEqual('string', packed_message.data) 

def testParseExotic(self, message_module):
    message = message_module.TestAllTypes()
    text = ('repeated_int64: -9223372036854775808\n'
            'repeated_uint64: 18446744073709551615\n'
            'repeated_double: 123.456\n'
            'repeated_double: 1.23e+22\n'
            'repeated_double: 1.23e-18\n'
            'repeated_string: \n'
            '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
            'repeated_string: "foo" \'corge\' "grault"\n'
            'repeated_string: "\\303\\274\\352\\234\\237"\n'
            'repeated_string: "\\xc3\\xbc"\n'
            'repeated_string: "\xc3\xbc"\n')
    text_format.Parse(text, message)

    self.assertEqual(-9223372036854775808, message.repeated_int64[0])
    self.assertEqual(18446744073709551615, message.repeated_uint64[0])
    self.assertEqual(123.456, message.repeated_double[0])
    self.assertEqual(1.23e22, message.repeated_double[1])
    self.assertEqual(1.23e-18, message.repeated_double[2])
    self.assertEqual('\000\001\a\b\f\n\r\t\v\\\'"', message.repeated_string[0])
    self.assertEqual('foocorgegrault', message.repeated_string[1])
    self.assertEqual('\u00fc\ua71f', message.repeated_string[2])
    self.assertEqual('\u00fc', message.repeated_string[3]) 

def main(unused_argv):
  request = inference_flags.request_from_flags()

  if not gfile.Exists(request.segmentation_output_dir):
    gfile.MakeDirs(request.segmentation_output_dir)

  bbox = bounding_box_pb2.BoundingBox()
  text_format.Parse(FLAGS.bounding_box, bbox)

  runner = inference.Runner()
  runner.start(request)
  runner.run((bbox.start.z, bbox.start.y, bbox.start.x),
             (bbox.size.z, bbox.size.y, bbox.size.x))

  counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt')
  if not gfile.Exists(counter_path):
    runner.counters.dump(counter_path) 

def LoadInferenceGraph(path, clear_device_placement=False):
  """Parse the given path as an InferenceGraph proto.

  Args:
    path: The path to the file to load.
    clear_device_placement: If true, clears device field from nodes in graph.

  Returns:
    An InferenceGraph object.
  """
  inference_graph = inference_graph_pb2.InferenceGraph()
  with tf.io.gfile.GFile(path, "r") as f:
    text_format.Parse(f.read(), inference_graph)
  if clear_device_placement:
    for node in inference_graph.graph_def.node:
      node.ClearField("device")
    for function in inference_graph.graph_def.library.function:
      for node_def in function.node_def:
        node_def.ClearField("device")
  return inference_graph 

def testParseStringFieldUnescape(self, message_module):
    message = message_module.TestAllTypes()
    text = r'''repeated_string: "\xf\x62"
               repeated_string: "\\xf\\x62"
               repeated_string: "\\\xf\\\x62"
               repeated_string: "\\\\xf\\\\x62"
               repeated_string: "\\\\\xf\\\\\x62"
               repeated_string: "\x5cx20"'''

    text_format.Parse(text, message)

    SLASH = '\\'
    self.assertEqual('\x0fb', message.repeated_string[0])
    self.assertEqual(SLASH + 'xf' + SLASH + 'x62', message.repeated_string[1])
    self.assertEqual(SLASH + '\x0f' + SLASH + 'b', message.repeated_string[2])
    self.assertEqual(SLASH + SLASH + 'xf' + SLASH + SLASH + 'x62',
                     message.repeated_string[3])
    self.assertEqual(SLASH + SLASH + '\x0f' + SLASH + SLASH + 'b',
                     message.repeated_string[4])
    self.assertEqual(SLASH + 'x20', message.repeated_string[5]) 

def get_proto(push_list, output_dir):
    bench_factory = aibench_pb2.BenchFactory()
    model_factory = aibench_pb2.ModelFactory()
    try:
        with open("aibench/proto/benchmark.meta", "rb") as fin:
            file_content = fin.read()
            text_format.Parse(file_content, bench_factory)
            filepath = output_dir + "/benchmark.pb"
            with open(filepath, "wb") as fout:
                fout.write(bench_factory.SerializeToString())
                push_list.append(filepath)
        with open("aibench/proto/model.meta", "rb") as fin:
            file_content = fin.read()
            text_format.Parse(file_content, model_factory)
            filepath = output_dir + "/model.pb"
            with open(filepath, "wb") as fout:
                fout.write(model_factory.SerializeToString())
                push_list.append(filepath)
    except text_format.ParseError as e:
        raise IOError("Cannot parse file.", e)

    return bench_factory, model_factory 

def testBulkFeatureIdExtractorFailsOnLinkedFeature(self):
    component_spec = spec_pb2.ComponentSpec()
    text_format.Parse("""
        name: "test"
        network_unit {
          registered_name: "IdentityNetwork"
        }
        fixed_feature {
          name: "fixed" embedding_dim: -1 size: 1
        }
        linked_feature {
          name: "linked" embedding_dim: -1 size: 1
          source_translator: "identity"
          source_component: "mock"
        }
        """, component_spec)
    with tf.Graph().as_default():
      with self.assertRaises(ValueError):
        unused_comp = bulk_component.BulkFeatureIdExtractorComponentBuilder(
            self.master, component_spec) 

def testBulkFeatureIdExtractorOkWithOneFixedFeature(self):
    component_spec = spec_pb2.ComponentSpec()
    text_format.Parse("""
        name: "test"
        network_unit {
          registered_name: "IdentityNetwork"
        }
        fixed_feature {
          name: "fixed" embedding_dim: -1 size: 1
        }
        """, component_spec)
    with tf.Graph().as_default():
      comp = bulk_component.BulkFeatureIdExtractorComponentBuilder(
          self.master, component_spec)

      # Should not raise errors.
      self.network_states[component_spec.name] = component.NetworkState()
      comp.build_greedy_training(self.master_state, self.network_states)
      self.network_states[component_spec.name] = component.NetworkState()
      comp.build_greedy_inference(self.master_state, self.network_states) 

def testParseStringFieldUnescape(self, message_module):
    message = message_module.TestAllTypes()
    text = r'''repeated_string: "\xf\x62"
               repeated_string: "\\xf\\x62"
               repeated_string: "\\\xf\\\x62"
               repeated_string: "\\\\xf\\\\x62"
               repeated_string: "\\\\\xf\\\\\x62"
               repeated_string: "\x5cx20"'''

    text_format.Parse(text, message)

    SLASH = '\\'
    self.assertEqual('\x0fb', message.repeated_string[0])
    self.assertEqual(SLASH + 'xf' + SLASH + 'x62', message.repeated_string[1])
    self.assertEqual(SLASH + '\x0f' + SLASH + 'b', message.repeated_string[2])
    self.assertEqual(SLASH + SLASH + 'xf' + SLASH + SLASH + 'x62',
                     message.repeated_string[3])
    self.assertEqual(SLASH + SLASH + '\x0f' + SLASH + SLASH + 'b',
                     message.repeated_string[4])
    self.assertEqual(SLASH + 'x20', message.repeated_string[5]) 

def testParseMessageSet(self):
    message = unittest_pb2.TestAllTypes()
    text = ('repeated_uint64: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('message_set {\n'
            '  [protobuf_unittest.TestMessageSetExtension1] {\n'
            '    i: 23\n'
            '  }\n'
            '  [protobuf_unittest.TestMessageSetExtension2] {\n'
            '    str: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str) 

def RunTraining(self, hyperparam_config):
    master_spec = self.LoadSpec('master_spec_link.textproto')

    self.assertTrue(isinstance(hyperparam_config, spec_pb2.GridPoint))
    gold_doc = sentence_pb2.Sentence()
    text_format.Parse(_DUMMY_GOLD_SENTENCE, gold_doc)
    gold_doc_2 = sentence_pb2.Sentence()
    text_format.Parse(_DUMMY_GOLD_SENTENCE_2, gold_doc_2)
    reader_strings = [
        gold_doc.SerializeToString(), gold_doc_2.SerializeToString()
    ]
    tf.logging.info('Generating graph with config: %s', hyperparam_config)
    with tf.Graph().as_default():
      builder = graph_builder.MasterBuilder(master_spec, hyperparam_config)

      target = spec_pb2.TrainTarget()
      target.name = 'testTraining-all'
      train = builder.add_training_from_config(target)
      with self.test_session() as sess:
        logging.info('Initializing')
        sess.run(tf.global_variables_initializer())

        # Run one iteration of training and verify nothing crashes.
        logging.info('Training')
        sess.run(train['run'], feed_dict={train['input_batch']: reader_strings}) 

def testParseGroupNotClosed(self):
    message = unittest_pb2.TestAllTypes()
    text = 'RepeatedGroup: <'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected ">".',
                          text_format.Parse, text, message)
    text = 'RepeatedGroup: {'
    six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected "}".',
                          text_format.Parse, text, message) 

def setUpClass(cls):
    # At the moment, reference cycles between DescriptorPool and Message classes
    # are not detected and these objects are never freed.
    # To avoid errors with ReferenceLeakChecker, we create the class only once.
    file_desc = """
      name: "f/f.msg2"
      package: "f"
      message_type {
        name: "msg1"
        field {
          name: "payload"
          number: 1
          label: LABEL_OPTIONAL
          type: TYPE_STRING
        }
      }
      message_type {
        name: "msg2"
        field {
          name: "field"
          number: 1
          label: LABEL_OPTIONAL
          type: TYPE_MESSAGE
          type_name: "msg1"
        }
      }
    """
    pool = descriptor_pool.DescriptorPool()
    desc = descriptor_pb2.FileDescriptorProto()
    text_format.Parse(file_desc, desc)
    pool.Add(desc)
    cls.proto_cls = message_factory.MessageFactory(pool).GetPrototype(
        pool.FindMessageTypeByName('f.msg2')) 

def testParseOneof(self, message_module):
    m = message_module.TestAllTypes()
    m.oneof_uint32 = 11
    m2 = message_module.TestAllTypes()
    text_format.Parse(text_format.MessageToString(m), m2)
    self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) 

def assertImportFromName(self, msg, base_name):
    # Parse <type 'module.class_name'> to extra 'some.name' as a string.
    tp_name = str(type(msg)).split("'")[1]
    valid_names = ('Repeated%sContainer' % base_name,
                   'Repeated%sFieldContainer' % base_name)
    self.assertTrue(any(tp_name.endswith(v) for v in valid_names),
                    '%r does end with any of %r' % (tp_name, valid_names))

    parts = tp_name.split('.')
    class_name = parts[-1]
    module_name = '.'.join(parts[:-1])
    __import__(module_name, fromlist=[class_name]) 

def testParseMessageByFieldNumber(self):
    message = unittest_pb2.TestAllTypes()
    text = ('34: 1\n' 'repeated_uint64: 2\n')
    text_format.Parse(text, message, allow_field_number=True)
    self.assertEqual(1, message.repeated_uint64[0])
    self.assertEqual(2, message.repeated_uint64[1])

    message = unittest_mset_pb2.TestMessageSetContainer()
    text = ('1 {\n'
            '  1545008 {\n'
            '    15: 23\n'
            '  }\n'
            '  1547769 {\n'
            '    25: \"foo\"\n'
            '  }\n'
            '}\n')
    text_format.Parse(text, message, allow_field_number=True)
    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
    self.assertEqual(23, message.message_set.Extensions[ext1].i)
    self.assertEqual('foo', message.message_set.Extensions[ext2].str)

    # Can't parse field number without set allow_field_number=True.
    message = unittest_pb2.TestAllTypes()
    text = '34:1\n'
    six.assertRaisesRegex(self, text_format.ParseError, (
        r'1:1 : Message type "\w+.TestAllTypes" has no field named '
        r'"34".'), text_format.Parse, text, message)

    # Can't parse if field number is not found.
    text = '1234:1\n'
    six.assertRaisesRegex(
        self,
        text_format.ParseError,
        (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
         r'"1234".'),
        text_format.Parse,
        text,
        message,
        allow_field_number=True) 

def get_ops_and_kernels(proto_fileformat, proto_files, default_ops_str):
  """Gets the ops and kernels needed from the model files."""
  ops = set()

  for proto_file in proto_files:
    tf_logging.info('Loading proto file %s', proto_file)
    # Load GraphDef.
    file_data = gfile.GFile(proto_file, 'rb').read()
    if proto_fileformat == 'rawproto':
      graph_def = graph_pb2.GraphDef.FromString(file_data)
    else:
      assert proto_fileformat == 'textproto'
      graph_def = text_format.Parse(file_data, graph_pb2.GraphDef())

    # Find all ops and kernels used by the graph.
    for node_def in graph_def.node:
      if not node_def.device:
        node_def.device = '/cpu:0'
      kernel_class = pywrap_tensorflow.TryFindKernelClass(
          node_def.SerializeToString())
      if kernel_class:
        op_and_kernel = (str(node_def.op), kernel_class.decode('utf-8'))
        if op_and_kernel not in ops:
          ops.add(op_and_kernel)
      else:
        print(
            'Warning: no kernel found for op %s' % node_def.op, file=sys.stderr)

  # Add default ops.
  if default_ops_str and default_ops_str != 'all':
    for s in default_ops_str.split(','):
      op, kernel = s.split(':')
      op_and_kernel = (op, kernel)
      if op_and_kernel not in ops:
        ops.add(op_and_kernel)

  return list(sorted(ops)) 

def testParseGolden(self):
    golden_text = '\n'.join(self.ReadGolden(
        'text_format_unittest_data_oneof_implemented.txt'))
    parsed_message = unittest_pb2.TestAllTypes()
    r = text_format.Parse(golden_text, parsed_message)
    self.assertIs(r, parsed_message)

    message = unittest_pb2.TestAllTypes()
    test_util.SetAllFields(message)
    self.assertEqual(message, parsed_message) 

def testTopAnyMessage(self):
    packed_msg = unittest_pb2.OneString()
    msg = any_pb2.Any()
    msg.Pack(packed_msg)
    text = text_format.MessageToString(msg)
    other_msg = text_format.Parse(text, any_pb2.Any())
    self.assertEqual(msg, other_msg) 

def testParseMap(self):
    text = ('map_int32_int32 {\n'
            '  key: -123\n'
            '  value: -456\n'
            '}\n'
            'map_int64_int64 {\n'
            '  key: -8589934592\n'
            '  value: -17179869184\n'
            '}\n'
            'map_uint32_uint32 {\n'
            '  key: 123\n'
            '  value: 456\n'
            '}\n'
            'map_uint64_uint64 {\n'
            '  key: 8589934592\n'
            '  value: 17179869184\n'
            '}\n'
            'map_string_string {\n'
            '  key: "abc"\n'
            '  value: "123"\n'
            '}\n'
            'map_int32_foreign_message {\n'
            '  key: 111\n'
            '  value {\n'
            '    c: 5\n'
            '  }\n'
            '}\n')
    message = map_unittest_pb2.TestMap()
    text_format.Parse(text, message)

    self.assertEqual(-456, message.map_int32_int32[-123])
    self.assertEqual(-2**34, message.map_int64_int64[-2**33])
    self.assertEqual(456, message.map_uint32_uint32[123])
    self.assertEqual(2**34, message.map_uint64_uint64[2**33])
    self.assertEqual('123', message.map_string_string['abc'])
    self.assertEqual(5, message.map_int32_foreign_message[111].c) 

def testParseEmptyGroup(self):
    message = unittest_pb2.TestAllTypes()
    text = 'OptionalGroup: {}'
    text_format.Parse(text, message)
    self.assertTrue(message.HasField('optionalgroup'))

    message.Clear()

    message = unittest_pb2.TestAllTypes()
    text = 'OptionalGroup: <>'
    text_format.Parse(text, message)
    self.assertTrue(message.HasField('optionalgroup'))

  # Maps aren't really proto2-only, but our test schema only has maps for
  # proto2. 

def testParseBadIdentifier(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { "bb": 1 }')
    with self.assertRaises(text_format.ParseError) as e:
      text_format.Parse(text, message)
    self.assertEqual(str(e.exception),
                     '1:27 : Expected identifier or number, got "bb".') 

def testParseDuplicateScalars(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_int32: 42 ' 'optional_int32: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
        'have multiple "optional_int32" fields.'), text_format.Parse, text,
                          message) 

def testParseDuplicateMessages(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { bb: 1 } '
            'optional_nested_message { bb: 2 }')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:59 : Message type "protobuf_unittest.TestAllTypes" '
        'should not have multiple "optional_nested_message" fields.'),
                          text_format.Parse, text,
                          message) 

def testParseDuplicateExtensionScalars(self):
    message = unittest_pb2.TestAllExtensions()
    text = ('[protobuf_unittest.optional_int32_extension]: 42 '
            '[protobuf_unittest.optional_int32_extension]: 67')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:96 : Message type "protobuf_unittest.TestAllExtensions" '
        'should not have multiple '
        '"protobuf_unittest.optional_int32_extension" extensions.'),
                          text_format.Parse, text, message) 

def testParseNumericUnknownEnum(self):
    message = unittest_pb2.TestAllTypes()
    text = 'optional_nested_enum: 100'
    six.assertRaisesRegex(self, text_format.ParseError,
                          (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" '
                           r'has no value with number 100.'), text_format.Parse,
                          text, message) 

def testParseBadExtension(self):
    message = unittest_pb2.TestAllExtensions()
    text = '[unknown_extension]: 8\n'
    six.assertRaisesRegex(self, text_format.ParseError,
                          '1:2 : Extension "unknown_extension" not registered.',
                          text_format.Parse, text, message)
    message = unittest_pb2.TestAllTypes()
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
        'extensions.'), text_format.Parse, text, message) 

def testParseDuplicateNestedMessageScalars(self):
    message = unittest_pb2.TestAllTypes()
    text = ('optional_nested_message { bb: 1 } '
            'optional_nested_message { bb: 2 }')
    six.assertRaisesRegex(self, text_format.ParseError, (
        '1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
        'should not have multiple "bb" fields.'), text_format.Parse, text,
                          message)