Python Examples of tensorflow.core.example.example

Source File: graph2otherformat.py From structured-neural-summarization with MIT License

6 votes

def gettothepoint_converter():
    vocab_counter = Counter()

    def converter(graph:Dict, summary: List[str])-> str:
        input_sequence = [graph['node_labels'][i].lower() for i in graph['backbone_sequence']]
        assert len(input_sequence) > 0
        assert len(summary) > 0

        summary = [w.lower() for w in summary]
        if summary[-1] not in GET_TO_THE_POINT_END_TOKENS:
            summary.append('.')

        vocab_counter.update(input_sequence)
        vocab_counter.update(summary)

        tf_example = example_pb2.Example()
        tf_example.features.feature['article'].bytes_list.value.extend([' '.join(input_sequence).encode()])
        tf_example.features.feature['abstract'].bytes_list.value.extend([('<s>'+ ' '.join(summary) + '</s>').encode()])
        return tf_example.SerializeToString()

    return vocab_counter, converter

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def testDecodeExampleWithVarLenTensorToDense(self):
    np_array = np.array([[1, 2, 3], [4, 5, 6]])
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'labels': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'labels': parsing_ops.VarLenFeature(dtype=dtypes.int64),
      }
      items_to_handlers = {
          'labels': tfexample_decoder.Tensor(
              'labels', shape=np_array.shape),
      }
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels, np_array)

Source File: test_utils.py From lambda-packs with MIT License

6 votes

def generate_image(image_shape, image_format='jpeg', label=0):
  """Generates an image and an example containing the encoded image.

  GenerateImage must be called within an active session.

  Args:
    image_shape: the shape of the image to generate.
    image_format: the encoding format of the image.
    label: the int64 labels for the image.

  Returns:
    image: the generated image.
    example: a TF-example with a feature key 'image/encoded' set to the
      serialized image and a feature key 'image/format' set to the image
      encoding format ['jpeg', 'png'].
  """
  image = np.random.random_integers(0, 255, size=image_shape)
  tf_encoded = _encoder(image, image_format)
  example = example_pb2.Example(features=feature_pb2.Features(feature={
      'image/encoded': _encoded_bytes_feature(tf_encoded),
      'image/format': _string_feature(image_format),
      'image/class/label': _encoded_int64_feature(np.array(label)),
  }))

  return image, example.SerializeToString()

Source File: model_interface.py From justcopy-backend with MIT License

6 votes

def write_to_bin(self, news_dir, out_file):
        story_fnames = os.listdir(news_dir)
        num_stories = len(story_fnames)

        with open(out_file, 'wb') as writer:
            for idx,s in enumerate(story_fnames):
                #print(s)
                # Look in the tokenized story dirs to find the .story file corresponding to this url
                if os.path.isfile(os.path.join(news_dir, s)):
                    story_file = os.path.join(news_dir, s)
                article, abstract = self.get_art_abs(story_file)

                # Write to tf.Example
                if bytes(article, 'utf-8') == 0:
                    print('error!')
                tf_example = example_pb2.Example()
                tf_example.features.feature['article'].bytes_list.value.extend([bytes(article, 'utf-8') ])
                tf_example.features.feature['abstract'].bytes_list.value.extend([bytes(abstract, 'utf-8')])
                tf_example_str = tf_example.SerializeToString()
                str_len = len(tf_example_str)
                writer.write(struct.pack('q', str_len))
                writer.write(struct.pack('%ds' % str_len, tf_example_str))

        print("Finished writing file %s\n" % out_file)
        return story_fnames

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def testDecodeExampleWithInt64Tensor(self):
    np_array = np.random.randint(1, 10, size=(2, 3, 1))

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'array': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'array': parsing_ops.FixedLenFeature(np_array.shape, dtypes.int64)
      }
      items_to_handlers = {'array': tfexample_decoder.Tensor('array'),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_array] = decoder.decode(serialized_example, ['array'])
      self.assertAllEqual(tf_array.eval(), np_array)

Source File: dataset_test.py From spotify-tensorflow with Apache License 2.0

6 votes

def _write_test_data():
        schema = feature_spec_to_schema({"f0": tf.VarLenFeature(dtype=tf.int64),
                                         "f1": tf.VarLenFeature(dtype=tf.int64),
                                         "f2": tf.VarLenFeature(dtype=tf.int64)})
        batches = [
            [1, 4, None],
            [2, None, None],
            [3, 5, None],
            [None, None, None],
        ]

        example_proto = [example_pb2.Example(features=feature_pb2.Features(feature={
            "f" + str(i): feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=[f]))
            for i, f in enumerate(batch) if f is not None
        })) for batch in batches]

        return DataUtil.write_test_data(example_proto, schema)

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def GenerateImage(self, image_format, image_shape):
    """Generates an image and an example containing the encoded image.

    Args:
      image_format: the encoding format of the image.
      image_shape: the shape of the image to generate.

    Returns:
      image: the generated image.
      example: a TF-example with a feature key 'image/encoded' set to the
        serialized image and a feature key 'image/format' set to the image
        encoding format ['jpeg', 'JPEG', 'png', 'PNG', 'raw'].
    """
    num_pixels = image_shape[0] * image_shape[1] * image_shape[2]
    image = np.linspace(
        0, num_pixels - 1, num=num_pixels).reshape(image_shape).astype(np.uint8)
    tf_encoded = self._Encoder(image, image_format)
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'image/encoded': self._EncodedBytesFeature(tf_encoded),
        'image/format': self._StringFeature(image_format)
    }))

    return image, example.SerializeToString()

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def testDecodeExampleWithFloatTensor(self):
    np_array = np.random.rand(2, 3, 1).astype('f')

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'array': self._EncodedFloatFeature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'array': parsing_ops.FixedLenFeature(np_array.shape, dtypes.float32)
      }
      items_to_handlers = {'array': tfexample_decoder.Tensor('array'),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_array] = decoder.decode(serialized_example, ['array'])
      self.assertAllEqual(tf_array.eval(), np_array)

Source File: data_convert_example.py From yolo_v2 with Apache License 2.0

6 votes

def _binary_to_text():
  reader = open(FLAGS.in_file, 'rb')
  writer = open(FLAGS.out_file, 'w')
  while True:
    len_bytes = reader.read(8)
    if not len_bytes:
      sys.stderr.write('Done reading\n')
      return
    str_len = struct.unpack('q', len_bytes)[0]
    tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
    tf_example = example_pb2.Example.FromString(tf_example_str)
    examples = []
    for key in tf_example.features.feature:
      examples.append('%s=%s' % (key, tf_example.features.feature[key].bytes_list.value[0]))
    writer.write('%s\n' % '\t'.join(examples))
  reader.close()
  writer.close()

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def DecodeExample(self, serialized_example, item_handler, image_format):
    """Decodes the given serialized example with the specified item handler.

    Args:
      serialized_example: a serialized TF example string.
      item_handler: the item handler used to decode the image.
      image_format: the image format being decoded.

    Returns:
      the decoded image found in the serialized Example.
    """
    serialized_example = array_ops.reshape(serialized_example, shape=[])
    decoder = tfexample_decoder.TFExampleDecoder(
        keys_to_features={
            'image/encoded':
                parsing_ops.FixedLenFeature(
                    (), dtypes.string, default_value=''),
            'image/format':
                parsing_ops.FixedLenFeature(
                    (), dtypes.string, default_value=image_format),
        },
        items_to_handlers={'image': item_handler})
    [tf_image] = decoder.decode(serialized_example, ['image'])
    return tf_image

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def testDecodeExampleWithFixLenTensorWithShape(self):
    np_array = np.array([[1, 2, 3], [4, 5, 6]])

    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'labels': self._EncodedInt64Feature(np_array),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'labels':
              parsing_ops.FixedLenFeature(
                  np_array.shape, dtype=dtypes.int64),
      }
      items_to_handlers = {
          'labels': tfexample_decoder.Tensor(
              'labels', shape=np_array.shape),
      }
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels, np_array)

Source File: tfexample_decoder_test.py From auto-alt-text-lambda-api with MIT License

6 votes

def testDecodeExampleWithSparseTensor(self):
    np_indices = np.array([[1], [2], [5]])
    np_values = np.array([0.1, 0.2, 0.6]).astype('f')
    example = example_pb2.Example(features=feature_pb2.Features(feature={
        'indices': self._EncodedInt64Feature(np_indices),
        'values': self._EncodedFloatFeature(np_values),
    }))

    serialized_example = example.SerializeToString()

    with self.test_session():
      serialized_example = array_ops.reshape(serialized_example, shape=[])
      keys_to_features = {
          'indices': parsing_ops.VarLenFeature(dtype=dtypes.int64),
          'values': parsing_ops.VarLenFeature(dtype=dtypes.float32),
      }
      items_to_handlers = {'labels': tfexample_decoder.SparseTensor(),}
      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
                                                   items_to_handlers)
      [tf_labels] = decoder.decode(serialized_example, ['labels'])
      labels = tf_labels.eval()
      self.assertAllEqual(labels.indices, np_indices)
      self.assertAllEqual(labels.values, np_values)
      self.assertAllEqual(labels.dense_shape, np_values.shape)

Source File: tf_example_decoder_test.py From Person-Detection-and-Tracking with MIT License

6 votes

def testDecodeObjectWeight(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    object_weights = [0.75, 1.0]
    example = tf.train.Example(features=tf.train.Features(
        feature={
            'image/encoded': self._BytesFeature(encoded_jpeg),
            'image/format': self._BytesFeature('jpeg'),
            'image/object/weight': self._FloatFeature(object_weights),
        })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[
        fields.InputDataFields.groundtruth_weights].get_shape().as_list()),
                        [None])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual(
        object_weights,
        tensor_dict[fields.InputDataFields.groundtruth_weights])

Source File: tf_example_decoder_test.py From Person-Detection-and-Tracking with MIT License

6 votes

def testDecodeObjectGroupOf(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    object_group_of = [0, 1]
    example = tf.train.Example(features=tf.train.Features(
        feature={
            'image/encoded': self._BytesFeature(encoded_jpeg),
            'image/format': self._BytesFeature('jpeg'),
            'image/object/group_of': self._Int64Feature(object_group_of),
        })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[
        fields.InputDataFields.groundtruth_group_of].get_shape().as_list()),
                        [2])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual(
        [bool(item) for item in object_group_of],
        tensor_dict[fields.InputDataFields.groundtruth_group_of])

Source File: tf_example_decoder_test.py From Person-Detection-and-Tracking with MIT License

6 votes

def testDecodeObjectIsCrowd(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    object_is_crowd = [0, 1]
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/encoded': self._BytesFeature(encoded_jpeg),
        'image/format': self._BytesFeature('jpeg'),
        'image/object/is_crowd': self._Int64Feature(object_is_crowd),
    })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[
        fields.InputDataFields.groundtruth_is_crowd].get_shape().as_list()),
                        [2])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual([bool(item) for item in object_is_crowd],
                        tensor_dict[
                            fields.InputDataFields.groundtruth_is_crowd])

Source File: tf_example_decoder_test.py From Person-Detection-and-Tracking with MIT License

6 votes

def testDecodeObjectArea(self):
    image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
    encoded_jpeg = self._EncodeImage(image_tensor)
    object_area = [100., 174.]
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/encoded': self._BytesFeature(encoded_jpeg),
        'image/format': self._BytesFeature('jpeg'),
        'image/object/area': self._FloatFeature(object_area),
    })).SerializeToString()

    example_decoder = tf_example_decoder.TfExampleDecoder()
    tensor_dict = example_decoder.decode(tf.convert_to_tensor(example))

    self.assertAllEqual((tensor_dict[fields.InputDataFields.groundtruth_area].
                         get_shape().as_list()), [2])
    with self.test_session() as sess:
      tensor_dict = sess.run(tensor_dict)

    self.assertAllEqual(object_area,
                        tensor_dict[fields.InputDataFields.groundtruth_area])