Python tensorflow.compat.v2.int64() Examples

def _info(self):
        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION,
            features=tfds.features.FeaturesDict(
                {
                    "images": {
                        "clean": tfds.features.Tensor(
                            shape=[224, 224, 3], dtype=tf.uint8
                        ),
                        "adversarial": tfds.features.Tensor(
                            shape=[224, 224, 3], dtype=tf.uint8
                        ),
                    },
                    "label": tfds.features.Tensor(shape=(), dtype=tf.int64),
                }
            ),
            supervised_keys=("images", "label"),
        ) 

def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "speech": tfds.features.Audio(
                file_format="wav", sample_rate=24000),
            "text_original": tfds.features.Text(),
            "text_normalized": tfds.features.Text(),
            "speaker_id": tf.int64,
            "chapter_id": tf.int64,
            "id": tf.string,
        }),
        supervised_keys=("text_normalized", "speech"),
        homepage=_URL,
        citation=_CITATION,
        metadata=tfds.core.MetadataDict(sample_rate=24000,),
    ) 

def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "speech":
                tfds.features.Audio(sample_rate=16000),
            "text":
                tfds.features.Text(
                    encoder_config=self.builder_config.text_encoder_config),
            "speaker_id":
                tf.int64,
            "chapter_id":
                tf.int64,
            "id":
                tf.string,
        }),
        supervised_keys=("speech", "text"),
        homepage=_URL,
        citation=_CITATION,
        metadata=tfds.core.MetadataDict(sample_rate=16000,),
    ) 

def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description="Dataset with images from 5 classes (see config name for "
        "information on the specific class)",
        features=tfds.features.FeaturesDict({
            "image": tfds.features.Image(),
            "image/filename": tfds.features.Text(),
            "PetID": tfds.features.Text(),
            "attributes": {name: tf.int64 for name in _INT_FEATS},
            "label": tfds.features.ClassLabel(num_classes=5),
        }),
        supervised_keys=("attributes", "label"),
        homepage="https://www.kaggle.com/c/petfinder-adoption-prediction/data",
        citation=_CITATION,
    ) 

def _parse_single_image(self, example_proto):
    """Parses single video from the input tfrecords.

    Args:
      example_proto: tfExample proto with a single video.

    Returns:
      dict with all frames, positions and actions.
    """

    feature_map = {
        "image": tf.io.FixedLenFeature(shape=[], dtype=tf.string),
        "filename": tf.io.FixedLenFeature(shape=[], dtype=tf.string),
        "label": tf.io.FixedLenFeature(shape=[], dtype=tf.int64),
    }

    parse_single = tf.io.parse_single_example(example_proto, feature_map)

    return parse_single 

def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "image":
                tfds.features.Image(shape=(105, 105, 3), encoding_format="png"),
            "alphabet":
                tfds.features.ClassLabel(num_classes=_NUM_ALPHABETS),
            "alphabet_char_id":
                tf.int64,
            "label":
                tfds.features.ClassLabel(num_classes=_NUM_CLASSES),
        }),
        supervised_keys=("image", "label"),
        homepage=_BASE_URL,
        citation=_CITATION,
    ) 

def testMultiDimensionalShape(self):
    """Check that stateless_random_shuffle works with multi-dim shapes."""
    for dtype in (tf.int32, tf.int64, tf.float32, tf.float64):
      input_permutation = tf.constant([[[1], [2], [3]], [[4], [5], [6]]],
                                      dtype=dtype)
      random_shuffle = tff_rnd.stateless_random_shuffle(
          input_permutation, seed=(1, 42))
      random_permutation_first_call = self.evaluate(random_shuffle)
      random_permutation_next_call = self.evaluate(random_shuffle)
      input_permutation = self.evaluate(input_permutation)
      # Check that the dtype is correct
      np.testing.assert_equal(random_permutation_first_call.dtype,
                              dtype.as_numpy_dtype)
      # Check that the shuffles are the same
      np.testing.assert_array_equal(random_permutation_first_call,
                                    random_permutation_next_call)
      # Check that the output shape is correct
      np.testing.assert_equal(random_permutation_first_call.shape,
                              input_permutation.shape) 

def testOutputIsStatelessSession(self):
    """Checks that stateless_random_shuffle is stateless across Sessions."""
    random_permutation_next_call = None
    for dtype in (tf.int32, tf.int64, tf.float32, tf.float64):
      random_permutation = tff_rnd.stateless_random_shuffle(
          tf.range(10, dtype=dtype), seed=tf.constant((100, 42), tf.int64))
      with tf.compat.v1.Session() as sess:
        random_permutation_first_call = sess.run(random_permutation)
      if random_permutation_next_call is not None:
        # Checks that the values are the same across different dtypes
        np.testing.assert_array_equal(random_permutation_first_call,
                                      random_permutation_next_call)
      with tf.compat.v1.Session() as sess:
        random_permutation_next_call = sess.run(random_permutation)
      np.testing.assert_array_equal(random_permutation_first_call,
                                    random_permutation_next_call) 

def testOutputIsIndependentOfInputValues(self):
    """stateless_random_shuffle output is independent of input_tensor values."""
    # Generate sorted array of random numbers to control that the result
    # is independent of `input_tesnor` values
    np.random.seed(25)
    random_input = np.random.normal(size=[10])
    random_input.sort()
    for dtype in (tf.int32, tf.int64, tf.float32, tf.float64):
      # Permutation of a sequence [0, 1, .., 9]
      random_permutation = tff_rnd.stateless_random_shuffle(
          tf.range(10, dtype=dtype), seed=(100, 42))
      random_permutation = self.evaluate(random_permutation)
      # Shuffle `random_input` with the same seed
      random_shuffle_control = tff_rnd.stateless_random_shuffle(
          random_input, seed=(100, 42))
      random_shuffle_control = self.evaluate(random_shuffle_control)
      # Checks that the generated permutation does not depend on the underlying
      # values
      np.testing.assert_array_equal(
          np.argsort(random_permutation), np.argsort(random_shuffle_control)) 

def testOutputIsPermutation(self):
    """Checks that stateless_random_shuffle outputs a permutation."""
    for dtype in (tf.int32, tf.int64, tf.float32, tf.float64):
      identity_permutation = tf.range(10, dtype=dtype)
      random_shuffle_seed_1 = tff_rnd.stateless_random_shuffle(
          identity_permutation, seed=tf.constant((1, 42), tf.int64))
      random_shuffle_seed_2 = tff_rnd.stateless_random_shuffle(
          identity_permutation, seed=tf.constant((2, 42), tf.int64))
      # Check that the shuffles are of the correct dtype
      for shuffle in (random_shuffle_seed_1, random_shuffle_seed_2):
        np.testing.assert_equal(shuffle.dtype, dtype.as_numpy_dtype)
      random_shuffle_seed_1 = self.evaluate(random_shuffle_seed_1)
      random_shuffle_seed_2 = self.evaluate(random_shuffle_seed_2)
      identity_permutation = self.evaluate(identity_permutation)
      # Check that the shuffles are different
      self.assertTrue(
          np.abs(random_shuffle_seed_1 - random_shuffle_seed_2).max())
      # Check that the shuffles are indeed permutations
      for shuffle in (random_shuffle_seed_1, random_shuffle_seed_2):
        self.assertAllEqual(set(shuffle), set(identity_permutation)) 

def _reshape_inner_dims(
    tensor: tf.Tensor,
    shape: tf.TensorShape,
    new_shape: tf.TensorShape) -> tf.Tensor:
  """Reshapes tensor to: shape(tensor)[:-len(shape)] + new_shape."""
  tensor_shape = tf.shape(tensor)
  ndims = shape.rank
  tensor.shape[-ndims:].assert_is_compatible_with(shape)
  new_shape_inner_tensor = tf.cast(
      [-1 if d is None else d for d in new_shape.as_list()], tf.int64)
  new_shape_outer_tensor = tf.cast(
      tensor_shape[:-ndims], tf.int64)
  full_new_shape = tf.concat(
      (new_shape_outer_tensor, new_shape_inner_tensor), axis=0)
  new_tensor = tf.reshape(tensor, full_new_shape)
  new_tensor.set_shape(tensor.shape[:-ndims] + new_shape)
  return new_tensor 

def setUp(self):
    super(LogicTest, self).setUp()
    self.array_transforms = [
        lambda x: x,  # Identity,
        tf.convert_to_tensor,
        np.array,
        lambda x: np.array(x, dtype=np.int32),
        lambda x: np.array(x, dtype=np.int64),
        lambda x: np.array(x, dtype=np.float32),
        lambda x: np.array(x, dtype=np.float64),
        array_ops.array,
        lambda x: array_ops.array(x, dtype=tf.int32),
        lambda x: array_ops.array(x, dtype=tf.int64),
        lambda x: array_ops.array(x, dtype=tf.float32),
        lambda x: array_ops.array(x, dtype=tf.float64),
    ] 

def testCumProdAndSum(self):

    def run_test(arr, *args, **kwargs):
      for fn in self.array_transforms:
        arg = fn(arr)
        self.match(
            array_ops.cumprod(arg, *args, **kwargs),
            np.cumprod(arg, *args, **kwargs))
        self.match(
            array_ops.cumsum(arg, *args, **kwargs),
            np.cumsum(arg, *args, **kwargs))

    run_test([])
    run_test([1, 2, 3])
    run_test([1, 2, 3], dtype=float)
    run_test([1, 2, 3], dtype=np.float32)
    run_test([1, 2, 3], dtype=np.float64)
    run_test([1., 2., 3.])
    run_test([1., 2., 3.], dtype=int)
    run_test([1., 2., 3.], dtype=np.int32)
    run_test([1., 2., 3.], dtype=np.int64)
    run_test([[1, 2], [3, 4]], axis=1)
    run_test([[1, 2], [3, 4]], axis=0)
    run_test([[1, 2], [3, 4]], axis=-1)
    run_test([[1, 2], [3, 4]], axis=-2) 

def _info(self):
        return tfds.core.DatasetInfo(
            builder=self,
            description=_DESCRIPTION,
            features=tfds.features.FeaturesDict(
                {
                    "speech": tfds.features.Audio(),
                    "text": tfds.features.Text(
                        encoder_config=self.builder_config.text_encoder_config
                    ),
                    "speaker_id": tf.int64,
                    "chapter_id": tf.int64,
                    "id": tf.string,
                    "label": tfds.features.ClassLabel(names=_LABELS),
                }
            ),
            supervised_keys=("speech", "label"),
            homepage=_URL,
            citation=_CITATION,
            metadata=tfds.core.MetadataDict(sample_rate=16000,),
        ) 

def _testBinOp(self, a, b, out, f, types=None):
    a = t2a(tf.convert_to_tensor(value=a, dtype=np.int32))
    b = t2a(tf.convert_to_tensor(value=b, dtype=np.int32))
    if not isinstance(out, arrays.ndarray):
      out = t2a(tf.convert_to_tensor(value=out, dtype=np.int32))
    if types is None:
      types = [[np.int32, np.int32, np.int32],
               [np.int64, np.int32, np.int64],
               [np.int32, np.int64, np.int64],
               [np.float32, np.int32, np.float64],
               [np.int32, np.float32, np.float64],
               [np.float32, np.float32, np.float32],
               [np.float64, np.float32, np.float64],
               [np.float32, np.float64, np.float64]]
    for a_type, b_type, out_type in types:
      o = f(a.astype(a_type), b.astype(b_type))
      self.assertIs(o.dtype.type, out_type)
      self.assertAllEqual(out.astype(out_type), o) 

def _key2seed(a):
  """Converts an RNG key to an RNG seed.

  Args:
    a: an RNG key, an ndarray of shape [] and dtype `np.int64`.

  Returns:
    an RNG seed, a tensor of shape [2] and dtype `tf.int32`.
  """

  def int64_to_int32s(a):
    """Converts an int64 tensor of shape [] to an int32 tensor of shape [2]."""
    a = tf.cast(a, tf.uint64)
    fst = tf.cast(a, tf.uint32)
    snd = tf.cast(
        tf.bitwise.right_shift(a, tf.constant(32, tf.uint64)), tf.uint32)
    a = [fst, snd]
    a = tf.nest.map_structure(lambda x: tf.cast(x, tf.int32), a)
    a = tf.stack(a)
    return a

  return int64_to_int32s(a.data) 

def true_divide(x1, x2):
  def _avoid_float64(x1, x2):
    if x1.dtype == x2.dtype and x1.dtype in (tf.int32, tf.int64):
      x1 = tf.cast(x1, dtype=tf.float32)
      x2 = tf.cast(x2, dtype=tf.float32)
    return x1, x2

  def f(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      float_ = dtypes.default_float_type()
      x1 = tf.cast(x1, float_)
      x2 = tf.cast(x2, float_)
    if not dtypes.is_allow_float64():
      # tf.math.truediv in Python3 produces float64 when both inputs are int32
      # or int64. We want to avoid that when is_allow_float64() is False.
      x1, x2 = _avoid_float64(x1, x2)
    return tf.math.truediv(x1, x2)
  return _bin_op(f, x1, x2) 

def testDenseFeaturesInKeras(self):
    features = {
        "text": np.array(["hello world", "pair-programming"]),
    }
    label = np.int64([0, 1])
    feature_columns = [
        hub.text_embedding_column_v2("text", self.model, trainable=True),
    ]
    input_features = dict(
        text=tf.keras.layers.Input(name="text", shape=[None], dtype=tf.string))
    dense_features = tf.keras.layers.DenseFeatures(feature_columns)
    x = dense_features(input_features)
    x = tf.keras.layers.Dense(16, activation="relu")(x)
    logits = tf.keras.layers.Dense(1, activation="linear")(x)
    model = tf.keras.Model(inputs=input_features, outputs=logits)
    model.compile(
        optimizer="rmsprop", loss="binary_crossentropy", metrics=["accuracy"])
    model.fit(x=features, y=label, epochs=10)
    self.assertAllEqual(model.predict(features["text"]).shape, [2, 1]) 

def _seed2key(a):
  """Converts an RNG seed to an RNG key.

  Args:
    a: an RNG seed, a tensor of shape [2] and dtype `tf.int32`.

  Returns:
    an RNG key, an ndarray of shape [] and dtype `np.int64`.
  """

  def int32s_to_int64(a):
    """Converts an int32 tensor of shape [2] to an int64 tensor of shape []."""
    a = tf.bitwise.bitwise_or(
        tf.cast(a[0], tf.uint64),
        tf.bitwise.left_shift(
            tf.cast(a[1], tf.uint64), tf.constant(32, tf.uint64)))
    a = tf.cast(a, tf.int64)
    return a

  return tf_np.asarray(int32s_to_int64(a)) 

def process_singledoc_dataset(dataset, batch_size, params):
  """Parses and batches single-doc dataset."""
  name_to_features = {
      "input_ids_a": tf.io.FixedLenFeature([params.len_title], tf.int64),
      "input_ids_b": tf.io.FixedLenFeature([params.len_passage], tf.int64),
      "input_mask_b": tf.io.FixedLenFeature([params.len_passage], tf.int64),
      "segment_ids_b": tf.io.FixedLenFeature([params.len_passage], tf.int64),
  }
  decode_fn = lambda record: decode_record(record, name_to_features)
  dataset = dataset.map(
      decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)

  def _select_data_from_record(record):
    """Filter out features to use for pretraining."""
    return {
        "input_ids": record["input_ids_b"],
        "input_mask": record["input_mask_b"],
        "segment_ids": record["segment_ids_b"],
        "target_ids": record["input_ids_a"],
    }

  dataset = dataset.map(
      _select_data_from_record,
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.batch(batch_size, drop_remainder=True)
  return dataset 

def iterator_single_var_loop(ds):
  s = tf.constant(0, dtype=tf.int64)
  for e in iter(ds):
    s = s * 10 + e
  return s 

def dataset_two_vars_loop(ds):
  s = tf.constant(0, dtype=tf.int64)
  p = tf.constant(1, dtype=tf.int64)
  for e in ds:
    s += e
    p *= e
  return s, p 

def _generate_examples(self, path):
        """Yields examples."""

        clean_key = "clean"
        adversarial_key = "adversarial"

        def _parse(serialized_example):
            ds_features = {
                "height": tf.io.FixedLenFeature([], tf.int64),
                "width": tf.io.FixedLenFeature([], tf.int64),
                "label": tf.io.FixedLenFeature([], tf.int64),
                "adv-image": tf.io.FixedLenFeature([], tf.string),
                "clean-image": tf.io.FixedLenFeature([], tf.string),
            }
            example = tf.io.parse_single_example(serialized_example, ds_features)

            img_clean = tf.io.decode_raw(example["clean-image"], tf.float32)
            img_adv = tf.io.decode_raw(example["adv-image"], tf.float32)
            # float values are integers in [0.0, 255.0] for clean and adversarial
            img_clean = tf.cast(img_clean, tf.uint8)
            img_clean = tf.reshape(img_clean, (example["height"], example["width"], 3))
            img_adv = tf.cast(img_adv, tf.uint8)
            img_adv = tf.reshape(img_adv, (example["height"], example["width"], 3))
            return {clean_key: img_clean, adversarial_key: img_adv}, example["label"]

        ds = tf.data.TFRecordDataset(filenames=[path])
        ds = ds.map(lambda x: _parse(x))
        default_graph = tf.compat.v1.keras.backend.get_session().graph
        ds = tfds.as_numpy(ds, graph=default_graph)

        for i, (img, label) in enumerate(ds):
            yield str(i), {
                "images": img,
                "label": label,
            } 

def multidoc_parse_spec(params, training=True):
  """Gets the mutli-doc tf.Example parsing spec."""
  len_p = params.len_passage
  name_to_features = {}
  feature_list = ["input_ids", "input_mask", "segment_ids"]
  for idx in params.passage_list:
    for feature in feature_list:
      name_to_features["%s_%s" % (feature, idx)] = tf.io.FixedLenFeature(
          [len_p], tf.int64)
  if training:
    # Cluster title.
    name_to_features["input_ids_a"] = tf.io.FixedLenFeature([params.len_title],
                                                            tf.int64)
  return name_to_features, feature_list 

def decode_sparse_record(record, name_to_features):
  """Decodes a sparse record to a TensorFlow example."""
  example = tf.io.parse_single_example(record, name_to_features)

  # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
  # So cast all int64 to int32.
  for name in list(example.keys()):
    t = example[name]
    if t.dtype == tf.int64:
      t = tf.cast(t, tf.int32)
    example[name] = tf.sparse.to_dense(t)

  return example 

def iterator_two_vars_loop(ds):
  s = tf.constant(0, dtype=tf.int64)
  p = tf.constant(1, dtype=tf.int64)
  for e in iter(ds):
    s += e
    p *= e
  return s, p 

def dataset_loop_with_break(ds):
  s = tf.constant(0, dtype=tf.int64)
  for e in ds:
    s = s * 10 + e
    if s > 100:
      break
  return s 

def iterator_resuming_loop(ds):
  s = tf.constant(0, dtype=tf.int64)
  itr = iter(ds)
  for e in itr:
    s = s * 10 + e
    break
  for e in itr:
    s = s * 10 + e
    break
  for e in itr:
    s = s * 10 + e
  return s 

def dataset_loop_with_return(ds):
  y = tf.constant(0, dtype=tf.int64)
  for e in ds:
    y = e
    return y
  return y 

def decode_record(record, name_to_features):
  """Decodes a record to a TensorFlow example."""
  example = tf.io.parse_single_example(record, name_to_features)

  # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
  # So cast all int64 to int32.
  for name in list(example.keys()):
    t = example[name]
    if t.dtype == tf.int64:
      t = tf.cast(t, tf.int32)
    example[name] = t

  return example