Python tensorflow.QueueBase() Examples

The following are 11 code examples of tensorflow.QueueBase(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module tensorflow , or try the search function .
Example #1
Source File: trainer.py    From VDAIC2017 with MIT License 6 votes vote down vote up 
def __init__(self, config, input_queue=None, predict_tower=None):
        """
        :param config: a `TrainConfig` instance
        :param input_queue: a `tf.QueueBase` instance to be used to buffer datapoints.
            Defaults to a FIFO queue of size 100.
        :param predict_tower: list of gpu relative idx to run prediction. default to be [0].
            Use -1 for cpu.
        """
        super(QueueInputTrainer, self).__init__(config)
        self.input_vars = self.model.get_input_vars()
        # use a smaller queue size for now, to avoid https://github.com/tensorflow/tensorflow/issues/2942
        if input_queue is None:
            self.input_queue = tf.FIFOQueue(
                    50, [x.dtype for x in self.input_vars], name='input_queue')
        else:
            self.input_queue = input_queue

        # by default, use the first training gpu for prediction
        self.predict_tower = predict_tower or [0]
        self.dequed_inputs = None
Example #2
Source File: trainer.py    From ternarynet with Apache License 2.0 6 votes vote down vote up 
def __init__(self, config, input_queue=None, predict_tower=None):
        """
        :param config: a `TrainConfig` instance
        :param input_queue: a `tf.QueueBase` instance to be used to buffer datapoints.
            Defaults to a FIFO queue of size 100.
        :param predict_tower: list of gpu relative idx to run prediction. default to be [0].
            Use -1 for cpu.
        """
        super(QueueInputTrainer, self).__init__(config)
        self.input_vars = self.model.get_input_vars()
        # use a smaller queue size for now, to avoid https://github.com/tensorflow/tensorflow/issues/2942
        if input_queue is None:
            self.input_queue = tf.FIFOQueue(
                    50, [x.dtype for x in self.input_vars], name='input_queue')
        else:
            self.input_queue = input_queue

        # by default, use the first training gpu for prediction
        self.predict_tower = predict_tower or [0]
        self.dequed_inputs = None
Example #3
Source File: trainer.py    From DDRL with Apache License 2.0 5 votes vote down vote up 
def __init__(self, config, input_queue=None, predict_tower=None):
        """
        :param config: a `TrainConfig` instance
        :param input_queue: a `tf.QueueBase` instance to be used to buffer datapoints.
            Defaults to a FIFO queue of size 100.
        :param predict_tower: list of gpu relative idx to run prediction. default to be [0].
            Use -1 for cpu.
        """
        super(QueueInputTrainer, self).__init__(config)
        self.input_vars = self.model.get_input_vars()
        # use a smaller queue size for now, to avoid https://github.com/tensorflow/tensorflow/issues/2942


        queue_size = config.extra_arg['queue_size']
        self.dummy_predictor = config.extra_arg['dummy_predictor']
        print 'DUMMY PREDICTOR', self.dummy_predictor

        if input_queue is None:
            self.input_queue = tf.FIFOQueue(
                    queue_size, [x.dtype for x in self.input_vars], name='input_queue')
        else:
            self.input_queue = input_queue

        # by default, use the first training gpu for prediction
        self.predict_tower = predict_tower or [0]
        self.dequed_inputs = None


        self.queue_size_op = self.input_queue.size()
Example #4
Source File: input_source.py    From petridishnn with MIT License 5 votes vote down vote up 
def __init__(self, ds, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 50.
        """
        if not isinstance(ds, DataFlow):
            raise ValueError("QueueInput takes a DataFlow! Got {}".format(ds))
        self.queue = queue
        self.ds = ds
        self._inf_ds = RepeatedData(ds, -1)
        self._started = False
Example #5
Source File: input_source.py    From petridishnn with MIT License 5 votes vote down vote up 
def __init__(self, ds, batch_size, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            batch_size(int): the batch size.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 3000.
        """
        super(BatchQueueInput, self).__init__(ds, queue)
        self.batch_size = int(batch_size)
Example #6
Source File: input_source.py    From ADL with MIT License 5 votes vote down vote up 
def __init__(self, ds, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 50.
        """
        if not isinstance(ds, DataFlow):
            raise ValueError("QueueInput takes a DataFlow! Got {}".format(ds))
        self.queue = queue
        self.ds = ds
        self._inf_ds = RepeatedData(ds, -1)
        self._started = False
Example #7
Source File: input_source.py    From ADL with MIT License 5 votes vote down vote up 
def __init__(self, ds, batch_size, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            batch_size(int): the batch size.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 3000.
        """
        super(BatchQueueInput, self).__init__(ds, queue)
        self.batch_size = int(batch_size)
Example #8
Source File: plan_test.py    From fold with Apache License 2.0 5 votes vote down vote up 
def test_init_loom(self):
    p = plan.TrainPlan()
    p.compiler = block_compiler.Compiler().compile(blocks.Scalar())
    p.batch_size = 3
    p.task = 13
    p.num_dequeuers = 7

    self.assertRaisesWithLiteralMatch(
        ValueError, 'must have at least one PS task; 0', p.init_loom)

    p.ps_tasks = 5
    self.assertRaisesWithLiteralMatch(
        ValueError, 'worker_replicas must be at least num_queues + '
        'num_dequeuers; 0 vs. 5 + 7 = 12', p.init_loom)
    p.worker_replicas = 14

    # Would be best to actually create a queue and inspect it, but
    # tf.QueueBase doesn't currently expose these properties.
    self.assertEqual(p._create_queue(3, ctor=dict)['capacity'], 12)
    p.queue_capacity = 42
    q_dict = p._create_queue(3, ctor=dict)
    self.assertEqual(q_dict['capacity'], 42)
    self.assertEqual(q_dict['shared_name'], 'tensorflow_fold_plan_queue3')

    self.assertEqual(p.init_loom(), (True, False))

    p.compiler = block_compiler.Compiler().compile(blocks.Scalar())
    p.task = 3
    self.assertEqual(p.init_loom(), (False, True))

    p.compiler = block_compiler.Compiler().compile(blocks.Scalar())
    p.num_dequeuers = 0
    self.assertRaisesWithLiteralMatch(
        ValueError, 'cannot specify queue_capacity without also '
        'specifying num_dequeuers', p.init_loom)

    p.compiler = block_compiler.Compiler().compile(blocks.Scalar())
    p.queue_capacity = 0
    self.assertEqual(p.init_loom(), (True, True))
Example #9
Source File: trainer.py    From Distributed-BA3C with Apache License 2.0 5 votes vote down vote up 
def __init__(self, config, input_queue=None, predict_tower=None):
        """
        :param config: a `TrainConfig` instance
        :param input_queue: a `tf.QueueBase` instance to be used to buffer datapoints.
            Defaults to a FIFO queue of size 100.
        :param predict_tower: list of gpu relative idx to run prediction. default to be [0].
            Use -1 for cpu.
        """
        super(QueueInputTrainer, self).__init__(config)
        self.input_vars = self.model.get_input_vars()
        # use a smaller queue size for now, to avoid https://github.com/tensorflow/tensorflow/issues/2942


        queue_size = config.extra_arg['queue_size']
        self.dummy_predictor = config.extra_arg['dummy_predictor']
        print 'DUMMY PREDICTOR', self.dummy_predictor

        if input_queue is None:
            self.input_queue = tf.FIFOQueue(
                    queue_size, [x.dtype for x in self.input_vars], name='input_queue')
        else:
            self.input_queue = input_queue

        # by default, use the first training gpu for prediction
        self.predict_tower = predict_tower or [0]
        self.dequed_inputs = None


        self.queue_size_op = self.input_queue.size()
Example #10
Source File: input_source.py    From tensorpack with Apache License 2.0 5 votes vote down vote up 
def __init__(self, ds, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 50.
        """
        if not isinstance(ds, DataFlow):
            raise ValueError("QueueInput takes a DataFlow! Got {}".format(ds))
        self.queue = queue
        self.ds = ds
        self._inf_ds = RepeatedData(ds, -1)
        self._started = False
Example #11
Source File: input_source.py    From tensorpack with Apache License 2.0 5 votes vote down vote up 
def __init__(self, ds, batch_size, queue=None):
        """
        Args:
            ds(DataFlow): the input DataFlow.
            batch_size(int): the batch size.
            queue (tf.QueueBase): A :class:`tf.QueueBase` whose type
                should match the corresponding input signature of the model.
                Defaults to a FIFO queue of size 3000.
        """
        super(BatchQueueInput, self).__init__(ds, queue)
        self.batch_size = int(batch_size)