Python tensorflow.PaddingFIFOQueue() Examples

def testMultiQueueConstructor(self):
    with tf.Graph().as_default():
      q = tf.PaddingFIFOQueue(5, (tf.int32, tf.float32),
                              ((), ()),
                              shared_name="foo", name="Q")
    self.assertTrue(isinstance(q.queue_ref, tf.Tensor))
    self.assertEquals(tf.string_ref, q.queue_ref.dtype)
    self.assertProtoEquals("""
      name:'Q' op:'PaddingFIFOQueue'
      attr { key: 'component_types' value { list {
        type: DT_INT32 type : DT_FLOAT
      } } }
      attr { key: 'shapes' value { list { shape { } shape { } } } }
      attr { key: 'capacity' value { i: 5 } }
      attr { key: 'container' value { s: '' } }
      attr { key: 'shared_name' value { s: 'foo' } }
      """, q.queue_ref.op.node_def) 

def testParallelDequeueUpTo(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(1000, tf.float32, shapes=((),))
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_up_to(101)

      enqueue_op.run()
      close_op.run()

      # Dequeue up to 101 items in parallel on 10 threads, from closed queue.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

def testDequeueManyWithTensorParameter(self):
    with self.test_session():
      # Define a first queue that contains integer counts.
      dequeue_counts = [random.randint(1, 10) for _ in range(100)]
      count_q = tf.PaddingFIFOQueue(100, tf.int32, ((),))
      enqueue_counts_op = count_q.enqueue_many((dequeue_counts,))
      total_count = sum(dequeue_counts)

      # Define a second queue that contains total_count elements.
      elems = [random.randint(0, 100) for _ in range(total_count)]
      q = tf.PaddingFIFOQueue(total_count, tf.int32, ((),))
      enqueue_elems_op = q.enqueue_many((elems,))

      # Define a subgraph that first dequeues a count, then DequeuesMany
      # that number of elements.
      dequeued_t = q.dequeue_many(count_q.dequeue())

      enqueue_counts_op.run()
      enqueue_elems_op.run()

      dequeued_elems = []
      for _ in dequeue_counts:
        dequeued_elems.extend(dequeued_t.eval())
      self.assertEqual(elems, dequeued_elems) 

def testDequeueFromClosedQueue(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()
      close_op.run()
      for elem in elems:
        self.assertEqual([elem], dequeued_t.eval())

      # Expect the operation to fail due to the queue being closed.
      with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                   "is closed and has insufficient"):
        dequeued_t.eval() 

def get_input_queues(path, word2idx, batch_size=32, num_threads=8):
    input_ph = tf.placeholder(tf.int32, shape=[None])  # [T]
    queue = tf.PaddingFIFOQueue(shapes=[[None, ]], dtypes=[tf.int32], capacity=5000,)

    # TODO: enqueue_many would be faster, would require batch and padding at numpy-level
    enqueue_op = queue.enqueue([input_ph])
    def enqueue_data(sess):
        # for epoch in range(epoch_size):
        while True:  # 
            for idx, line in enumerate(read_data(path)):
                v = vectorize(line, word2idx)
                sess.run(enqueue_op, feed_dict={input_ph: v})

    # dequeue_batch = queue.dequeue_many(batch_size)
    dequeue_op = queue.dequeue()
    dequeue_batch = tf.train.batch([dequeue_op], batch_size=batch_size, num_threads=num_threads, capacity=1000, 
        dynamic_pad=True, name="batch_and_pad")

    # TODO: get corpus_size here
    return enqueue_data, dequeue_batch 

def testBlockingDequeueFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def dequeue():
        for elem in elems:
          self.assertEqual([elem], sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testDequeueUpToFromClosedQueueReturnsRemainder(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_up_to(3)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
        self.assertAllEqual(elems[3:], sess.run(dequeued_t))

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testBlockingDequeueManyFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_many(4)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems, sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testBlockingDequeueManyButNotAllFromClosedQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      close_op = q.close()
      dequeued_t = q.dequeue_many(3)

      enqueue_op.run()

      def dequeue():
        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testParallelDequeueMany(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(1000, tf.float32, shapes=((),))
      elems = [10.0 * x for x in range(1000)]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(100)

      enqueue_op.run()

      # Dequeue 100 items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue():
        dequeued_elems.extend(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems) 

def testEnqueueDequeueManyWrongPartiallyKnownShape(self):
    with self.test_session() as sess:
      # First dimension of second component is unknown, second
      # dimension must be 3.
      q = tf.PaddingFIFOQueue(10, (tf.int32, tf.int32), ((2, 2), (None, 3)))
      elems_ok = np.array([1] * 8).reshape((2, 2, 2)).astype(np.int32)
      elems_bad = tf.placeholder(tf.int32)
      enqueue_op = q.enqueue_many((elems_ok, elems_bad))
      dequeued_t = q.dequeue_many(2)
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError,
          "Shape mismatch in tuple component 1. "
          r"Expected \[2,\?,3\], got \[2,3,4\]"):
        sess.run([enqueue_op],
                 feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))})
        dequeued_t.eval() 

def testBlockingDequeueManyFromClosedEmptyQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      close_op = q.close()
      dequeued_t = q.dequeue_many(4)

      def dequeue():
        # Expect the operation to fail due to the queue being closed.
        with self.assertRaisesRegexp(tf.errors.OutOfRangeError,
                                     "is closed and has insufficient"):
          sess.run(dequeued_t)

      dequeue_thread = self.checkedThread(target=dequeue)
      dequeue_thread.start()
      # The close_op should run after the dequeue_thread has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      close_op.run()
      dequeue_thread.join() 

def testConstructorWithShapes(self):
    with tf.Graph().as_default():
      q = tf.PaddingFIFOQueue(5, (tf.int32, tf.float32),
                              shapes=(tf.TensorShape([1, 1, 2, 3]),
                                      tf.TensorShape([5, 8])), name="Q")
    self.assertTrue(isinstance(q.queue_ref, tf.Tensor))
    self.assertEquals(tf.string_ref, q.queue_ref.dtype)
    self.assertProtoEquals("""
      name:'Q' op:'PaddingFIFOQueue'
      attr { key: 'component_types' value { list {
        type: DT_INT32 type : DT_FLOAT
      } } }
      attr { key: 'shapes' value { list {
        shape { dim { size: 1 }
                dim { size: 1 }
                dim { size: 2 }
                dim { size: 3 } }
        shape { dim { size: 5 }
                dim { size: 8 } }
      } } }
      attr { key: 'capacity' value { i: 5 } }
      attr { key: 'container' value { s: '' } }
      attr { key: 'shared_name' value { s: '' } }
      """, q.queue_ref.op.node_def) 

def testBlockingEnqueueToFullQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(4, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      blocking_enqueue_op = q.enqueue((50.0,))
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def blocking_enqueue():
        sess.run(blocking_enqueue_op)
      thread = self.checkedThread(target=blocking_enqueue)
      thread.start()
      # The dequeue ops should run after the blocking_enqueue_op has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      for elem in elems:
        self.assertEqual([elem], dequeued_t.eval())
      self.assertEqual([50.0], dequeued_t.eval())
      thread.join() 

def testBlockingEnqueueManyToFullQueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(4, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0]
      enqueue_op = q.enqueue_many((elems,))
      blocking_enqueue_op = q.enqueue_many(([50.0, 60.0],))
      dequeued_t = q.dequeue()

      enqueue_op.run()

      def blocking_enqueue():
        sess.run(blocking_enqueue_op)
      thread = self.checkedThread(target=blocking_enqueue)
      thread.start()
      # The dequeue ops should run after the blocking_enqueue_op has blocked.
      # TODO(mrry): Figure out how to do this without sleeping.
      time.sleep(0.1)
      for elem in elems:
        self.assertEqual([elem], dequeued_t.eval())
        time.sleep(0.01)
      self.assertEqual([50.0], dequeued_t.eval())
      self.assertEqual([60.0], dequeued_t.eval()) 

def testParallelDequeue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Enqueue every element using a single thread.
      for enqueue_op in enqueue_ops:
        enqueue_op.run()

      # Run one consumer thread for each element in elems.
      results = []

      def dequeue():
        results.append(sess.run(dequeued_t))
      threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, results) 

def testParallelEnqueue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_ops = [q.enqueue((x,)) for x in elems]
      dequeued_t = q.dequeue()

      # Run one producer thread for each element in elems.
      def enqueue(enqueue_op):
        sess.run(enqueue_op)
      threads = [self.checkedThread(target=enqueue, args=(e,))
                 for e in enqueue_ops]
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()

      # Dequeue every element using a single thread.
      results = []
      for _ in xrange(len(elems)):
        results.append(dequeued_t.eval())
      self.assertItemsEqual(elems, results) 

def testMultiEnqueueManyWithPartiallyKnownShapes(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(
          10, (tf.float32, tf.int32), shapes=((), (None,)))
      float_elems = [10.0, 20.0, 30.0, 40.0]
      int_elems = [[1, 2], [3, 4], [5, 6], [7, 8]]
      enqueue_op = q.enqueue_many((float_elems, int_elems))
      dequeued_t = q.dequeue()

      enqueue_op.run()
      enqueue_op.run()

      for i in range(8):
        float_val, int_val = sess.run(dequeued_t)
        self.assertEqual(float_elems[i % 4], float_val)
        self.assertAllEqual(int_elems[i % 4], int_val) 

def testBigDequeueMany(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(2, tf.int32, ((),))
      elem = np.arange(4, dtype=np.int32)
      enq_list = [q.enqueue((e,)) for e in elem]
      deq = q.dequeue_many(4)

      results = []
      def blocking_dequeue():
        # Will only complete after 4 enqueues complete.
        results.extend(sess.run(deq))
      thread = self.checkedThread(target=blocking_dequeue)
      thread.start()
      # The dequeue should start and then block.
      for enq in enq_list:
        # TODO(mrry): Figure out how to do this without sleeping.
        time.sleep(0.1)
        self.assertEqual(len(results), 0)
        sess.run(enq)

      # Enough enqueued to unblock the dequeue
      thread.join()
      self.assertAllEqual(elem, results) 

def testSharedQueueSameSession(self):
    with self.test_session():
      q1 = tf.PaddingFIFOQueue(
          1, tf.float32, ((),), shared_name="shared_queue")
      q1.enqueue((10.0,)).run()

      q2 = tf.PaddingFIFOQueue(
          1, tf.float32, ((),), shared_name="shared_queue")

      q1_size_t = q1.size()
      q2_size_t = q2.size()

      self.assertEqual(q1_size_t.eval(), [1])
      self.assertEqual(q2_size_t.eval(), [1])

      self.assertEqual(q2.dequeue().eval(), [10.0])

      self.assertEqual(q1_size_t.eval(), [0])
      self.assertEqual(q2_size_t.eval(), [0])

      q2.enqueue((20.0,)).run()

      self.assertEqual(q1_size_t.eval(), [1])
      self.assertEqual(q2_size_t.eval(), [1])

      self.assertEqual(q1.dequeue().eval(), [20.0])

      self.assertEqual(q1_size_t.eval(), [0])
      self.assertEqual(q2_size_t.eval(), [0]) 

def testParallelEnqueueAndDequeue(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(50, tf.float32, shapes=((),))
      initial_elements = [10.0] * 49
      q.enqueue_many((initial_elements,)).run()

      enqueue_op = q.enqueue((20.0,))
      dequeued_t = q.dequeue()

      def enqueue():
        for _ in xrange(100):
          sess.run(enqueue_op)
      def dequeue():
        for _ in xrange(100):
          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))

      enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
      dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
      for enqueue_thread in enqueue_threads:
        enqueue_thread.start()
      for dequeue_thread in dequeue_threads:
        dequeue_thread.start()
      for enqueue_thread in enqueue_threads:
        enqueue_thread.join()
      for dequeue_thread in dequeue_threads:
        dequeue_thread.join()

      # Dequeue the initial count of elements to clean up.
      cleanup_elems = q.dequeue_many(49).eval()
      for elem in cleanup_elems:
        self.assertTrue(elem in (10.0, 20.0)) 

def testDoesNotLoseValue(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(1, tf.float32, ((),))
      enqueue_op = q.enqueue((10.0,))
      size_t = q.size()

      enqueue_op.run()
      for _ in range(500):
        self.assertEqual(size_t.eval(), [1]) 

def testMultiDequeueManyWithPartiallyKnownShapes(self):
    with self.test_session() as sess:
      q = tf.PaddingFIFOQueue(10, (tf.float32, tf.int32), shapes=((), (None,)))
      float_elems = [
          10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      int_elems = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10],
                   [11, 12], [13, 14], [15, 16], [17, 18], [19, 20]]
      enqueue_op = q.enqueue_many((float_elems, int_elems))
      dequeued_t = q.dequeue_many(4)
      dequeued_single_t = q.dequeue()

      enqueue_op.run()

      float_val, int_val = sess.run(dequeued_t)
      self.assertAllEqual(float_elems[0:4], float_val)
      self.assertAllEqual(int_elems[0:4], int_val)
      self.assertTrue(
          tf.TensorShape(float_val.shape).is_compatible_with(
              dequeued_t[0].get_shape()))
      self.assertTrue(
          tf.TensorShape(int_val.shape).is_compatible_with(
              dequeued_t[1].get_shape()))

      float_val, int_val = sess.run(dequeued_t)
      self.assertAllEqual(float_elems[4:8], float_val)
      self.assertAllEqual(int_elems[4:8], int_val)

      float_val, int_val = sess.run(dequeued_single_t)
      self.assertAllEqual(float_elems[8], float_val)
      self.assertAllEqual(int_elems[8], int_val)
      self.assertTrue(
          tf.TensorShape(float_val.shape).is_compatible_with(
              dequeued_single_t[0].get_shape()))
      self.assertTrue(
          tf.TensorShape(int_val.shape).is_compatible_with(
              dequeued_single_t[1].get_shape())) 

def testEnqueueWrongPartiallyKnownShapeAtRuntime(self):
    with self.test_session() as sess:
      # First dimension of second component is unknown, second
      # dimension must be 3.
      q = tf.PaddingFIFOQueue(10, (tf.int32, tf.int32), ((2, 2), (None, 3)))
      elems_ok = np.array([1] * 4).reshape((2, 2)).astype(np.int32)
      elems_bad = tf.placeholder(tf.int32)
      enqueue_op = q.enqueue((elems_ok, elems_bad))
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError, r"Expected \[\?,3\], got \[3,4\]"):
        sess.run([enqueue_op],
                 feed_dict={elems_bad: np.array([1] * 12).reshape((3, 4))}) 

def testEnqueueWrongType(self):
    q = tf.PaddingFIFOQueue(10, (tf.int32, tf.float32), ((), ()))

    with self.assertRaises(ValueError):
      q.enqueue((tf.placeholder(tf.int32), tf.placeholder(tf.int32)))

    with self.assertRaises(ValueError):
      q.enqueue_many((tf.placeholder(tf.int32), tf.placeholder(tf.int32))) 

def testEnqueueManyEmptyTypeConversion(self):
    q = tf.PaddingFIFOQueue(10, (tf.int32, tf.float32), ((), ()))
    enq = q.enqueue_many(([], []))
    self.assertEqual(tf.int32, enq.inputs[1].dtype)
    self.assertEqual(tf.float32, enq.inputs[2].dtype) 

def testBatchSizeMismatch(self):
    q = tf.PaddingFIFOQueue(10, (tf.int32, tf.int32, tf.int32), ((), (), ()))

    with self.assertRaises(ValueError):
      q.enqueue_many(([1, 2, 3], [1, 2], [1, 2, 3]))

    with self.assertRaises(ValueError):
      q.enqueue_many(([1, 2, 3], [1, 2], tf.placeholder(tf.int32)))

    with self.assertRaises(ValueError):
      q.enqueue_many((tf.placeholder(tf.int32), [1, 2], [1, 2, 3])) 

def testPartiallyKnownHighDimension(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(10, tf.int32, ((4, None, 4, None),))
      elems = np.array([[[[[x] * 4] * 4] * 4] * 4 for x in range(10)], np.int32)
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(10)

      enqueue_op.run()
      self.assertAllEqual(dequeued_t.eval(), elems) 

def testHighDimension(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(10, tf.int32, ((4, 4, 4, 4),))
      elems = np.array([[[[[x] * 4] * 4] * 4] * 4 for x in range(10)], np.int32)
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(10)

      enqueue_op.run()
      self.assertAllEqual(dequeued_t.eval(), elems) 

def testDequeueMany(self):
    with self.test_session():
      q = tf.PaddingFIFOQueue(10, tf.float32, ((),))
      elems = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
      enqueue_op = q.enqueue_many((elems,))
      dequeued_t = q.dequeue_many(4)

      enqueue_op.run()

      self.assertAllEqual(elems[0:4], dequeued_t.eval())
      self.assertAllEqual(elems[4:8], dequeued_t.eval())