Python fuel.datasets.IterableDataset() Examples

def test_training_interrupt():
    def process_batch(batch):
        time.sleep(0.1)

    algorithm = MockAlgorithm()
    algorithm.process_batch = process_batch

    main_loop = MockMainLoop(
        algorithm=algorithm,
        data_stream=IterableDataset(count()).get_example_stream(),
        extensions=[Printing()]
    )

    p = Process(target=main_loop.run)
    p.start()
    time.sleep(0.1)
    os.kill(p.pid, signal.SIGINT)
    time.sleep(0.1)
    assert p.is_alive()
    os.kill(p.pid, signal.SIGINT)
    time.sleep(0.2)
    assert not p.is_alive()
    p.join() 

def test_dataset_evaluators():
    X = theano.tensor.vector('X')
    Y = theano.tensor.vector('Y')

    data = [numpy.arange(1, 7, dtype=theano.config.floatX).reshape(3, 2),
            numpy.arange(11, 17, dtype=theano.config.floatX).reshape(3, 2)]
    data_stream = IterableDataset(dict(X=data[0],
                                       Y=data[1])).get_example_stream()

    validator = DatasetEvaluator([
        CrossEntropy(requires=[X, Y],
                     name="monitored_cross_entropy0"),
        # to test two same quantities and make sure that state will be reset
        CrossEntropy(requires=[X, Y],
                     name="monitored_cross_entropy1"),
        CategoricalCrossEntropy().apply(X, Y), ])
    values = validator.evaluate(data_stream)
    numpy.testing.assert_allclose(
        values['monitored_cross_entropy1'],
        values['categoricalcrossentropy_apply_cost']) 

def test_dataset_evaluators():
    X = theano.tensor.matrix('X')
    brick = TestBrick(name='test_brick')
    Y = brick.apply(X)
    graph = ComputationGraph([Y])
    monitor_variables = [v for v in graph.auxiliary_variables]
    validator = DatasetEvaluator(monitor_variables)

    data = [numpy.arange(1, 5, dtype=theano.config.floatX).reshape(2, 2),
            numpy.arange(10, 16, dtype=theano.config.floatX).reshape(3, 2)]
    data_stream = IterableDataset(dict(X=data)).get_example_stream()

    values = validator.evaluate(data_stream)
    assert values['test_brick_apply_V_squared'] == 4
    numpy.testing.assert_allclose(
        values['test_brick_apply_mean_row_mean'], numpy.vstack(data).mean())
    per_batch_mean = numpy.mean([batch.mean() for batch in data])
    numpy.testing.assert_allclose(
        values['test_brick_apply_mean_batch_element'], per_batch_mean)

    with assert_raises(Exception) as ar:
        data_stream = IterableDataset(dict(X2=data)).get_example_stream()
        validator.evaluate(data_stream)
    assert "Not all data sources" in ar.exception.args[0] 

def test_main_loop():
    old_config_profile_value = config.profile
    config.profile = True

    main_loop = MainLoop(
        MockAlgorithm(), IterableDataset(range(10)).get_example_stream(),
        extensions=[WriteBatchExtension(), FinishAfter(after_n_epochs=2)])
    main_loop.run()
    assert_raises(AttributeError, getattr, main_loop, 'model')

    assert main_loop.log.status['iterations_done'] == 20
    assert main_loop.log.status['_epoch_ends'] == [10, 20]
    assert len(main_loop.log) == 20
    for i in range(20):
        assert main_loop.log[i + 1]['batch'] == {'data': i % 10}

    config.profile = old_config_profile_value 

def test_sort_mapping_trivial_key(self):
        stream = DataStream(IterableDataset(self.data))
        transformer = Mapping(stream, SortMapping(operator.itemgetter(0)))
        assert_equal(list(transformer.get_epoch_iterator()),
                     list(zip([[1, 2, 3]] * 3))) 

def setUp(self):
        self.data = [1, 2, 3]
        self.stream = DataStream(IterableDataset(self.data)) 

def test_value_error_on_nonexistent_sources(self):
        def instantiate_dataset():
            return IterableDataset(self.data, sources=('dummy',))
        assert_raises(ValueError, instantiate_dataset) 

def test_default_transformer(self):
        class DoublingDataset(IterableDataset):
            def apply_default_transformer(self, stream):
                return Mapping(
                    stream, lambda sources: tuple(2 * s for s in sources))
        dataset = DoublingDataset(self.data)
        stream = dataset.apply_default_transformer(DataStream(dataset))
        assert_equal(list(stream.get_epoch_iterator()), [(2,), (4,), (6,)]) 

def test_no_axis_labels(self):
        assert IterableDataset(self.data).axis_labels is None 

def test_axis_labels(self):
        axis_labels = {'data': ('batch',)}
        dataset = IterableDataset(self.data, axis_labels=axis_labels)
        assert dataset.axis_labels == axis_labels 

def test_mapping_sort_multisource_ndarrays(self):
        data = OrderedDict([('x', numpy.array(self.data_x)),
                            ('y', numpy.array(self.data_y))])
        data_sorted = [(numpy.array([1, 2, 3]), numpy.array([6, 5, 4])),
                       (numpy.array([1, 2, 3]), numpy.array([4, 6, 5])),
                       (numpy.array([1, 2, 3]), numpy.array([4, 5, 6]))]
        stream = DataStream(IterableDataset(data))
        transformer = Mapping(
            stream, mapping=SortMapping(operator.itemgetter(0)))
        assert_equal(list(transformer.get_epoch_iterator()),
                     data_sorted) 

def test_add_sources(self):
        stream = DataStream(IterableDataset(self.data))
        transformer = Mapping(stream, lambda d: ([2 * i for i in d[0]],),
                              add_sources=('doubled',))
        assert_equal(transformer.sources, ('data', 'doubled'))
        assert_equal(list(transformer.get_epoch_iterator()),
                     list(zip(self.data, [[2, 4, 6], [4, 6, 2], [6, 4, 2]]))) 

def test_mapping_dict_add_sources(self):
        stream = DataStream(IterableDataset(self.data))
        transformer = Mapping(
            stream,
            lambda d: {'doubled': [2 * i for i in d['data']]},
            mapping_accepts=dict,
            add_sources=('doubled',))
        assert_equal(transformer.sources, ('data', 'doubled'))
        assert_equal(list(transformer.get_epoch_iterator()),
                     list(zip(self.data, [[2, 4, 6], [4, 6, 2], [6, 4, 2]]))) 

def setUp(self):
        stream = DataStream(IterableDataset(range(100)))
        self.transformer = Mapping(stream, lambda x: (x[0] + 1,)) 

def test_sort_mapping_alternate_key(self):
        stream = DataStream(IterableDataset(self.data))
        transformer = Mapping(stream, SortMapping(lambda x: -x[0]))
        assert_equal(list(transformer.get_epoch_iterator()),
                     list(zip([[3, 2, 1]] * 3))) 

def test_value_error_on_request(self):
        stream = DataStream(IterableDataset(self.data))
        transformer = Mapping(stream, lambda d: ([2 * i for i in d[0]],))
        assert_raises(ValueError, transformer.get_data, [0, 1]) 

def test_transform_source_batch_not_implemented(self):
        transformer = SourcewiseTransformer(
            DataStream(IterableDataset([1, 2])), True)
        assert_raises(
            NotImplementedError, transformer.transform_source_batch,
            None, 'foo') 

def test_ngram_stream_error_on_multiple_sources():
    # Check that NGram accepts only data streams with one source
    sentences = [list(numpy.random.randint(10, size=sentence_length))
                 for sentence_length in [3, 5, 7]]
    stream = DataStream(IterableDataset(sentences))
    stream.sources = ('1', '2')
    assert_raises(ValueError, NGrams, 4, stream) 

def test_ngram_stream():
    sentences = [list(numpy.random.randint(10, size=sentence_length))
                 for sentence_length in [3, 5, 7]]
    stream = DataStream(IterableDataset(sentences))
    ngrams = NGrams(4, stream)
    assert len(list(ngrams.get_epoch_iterator())) == 4 

def test_value_error_on_non_iterable(self):
        assert_raises(ValueError, IterableDataset, None) 

def test_reset_calls_reset_on_all_streams(self):
        streams = [FlagDataStream(IterableDataset([1, 2, 3])),
                   FlagDataStream(IterableDataset([4, 5, 6])),
                   FlagDataStream(IterableDataset([7, 8, 9]))]
        transformer = Merge(streams, ('1', '2', '3'))
        transformer.reset()
        assert all(stream.reset_called for stream in streams) 

def test_next_epoch_calls_next_epoch_on_all_streams(self):
        streams = [FlagDataStream(IterableDataset([1, 2, 3])),
                   FlagDataStream(IterableDataset([4, 5, 6])),
                   FlagDataStream(IterableDataset([7, 8, 9]))]
        transformer = Merge(streams, ('1', '2', '3'))
        transformer.next_epoch()
        assert all(stream.next_epoch_called for stream in streams) 

def setUp(self):
        self.streams = (
            DataStream(IterableDataset(['Hello world!'])),
            DataStream(IterableDataset(['Bonjour le monde!'])))
        self.batch_streams = (
            Batch(DataStream(IterableDataset(['Hello world!', 'Hi!'])),
                  iteration_scheme=ConstantScheme(2)),
            Batch(DataStream(IterableDataset(['Bonjour le monde!', 'Salut!'])),
                  iteration_scheme=ConstantScheme(2)))
        self.transformer = Merge(
            self.streams, ('english', 'french'))
        self.batch_transformer = Merge(
            self.batch_streams, ('english', 'french')) 

def test_value_error_on_example_stream(self):
        stream = DataStream(
            IterableDataset(
                dict(features=[[1], [2, 3]], targets=[[4, 5, 6], [7]])))
        assert_raises(ValueError, Padding, stream) 

def test_value_error_on_request(self):
        transformer = Padding(Batch(
            DataStream(
                IterableDataset(
                    dict(features=[[1], [2, 3]], targets=[[4, 5, 6], [7]]))),
            ConstantScheme(2)))
        assert_raises(ValueError, transformer.get_data, [0, 1]) 

def test_mask_sources(self):
        transformer = Padding(Batch(
            DataStream(
                IterableDataset(
                    OrderedDict([
                        ('features', [[1], [2, 3]]),
                        ('targets', [[4, 5, 6], [7]])]))),
            ConstantScheme(2)),
            mask_sources=('features',))
        assert_equal(len(next(transformer.get_epoch_iterator())), 3) 

def test_mask_dtype(self):
        transformer = Padding(Batch(
            DataStream(
                IterableDataset(
                    dict(features=[[1], [2, 3]], targets=[[4, 5, 6], [7]]))),
            ConstantScheme(2)),
            mask_dtype='uint8')
        assert_equal(
            str(next(transformer.get_epoch_iterator())[1].dtype), 'uint8') 

def test_2d_sequences_error_on_unequal_shapes(self):
        stream = Batch(
            DataStream(
                IterableDataset([numpy.ones((3, 4)), 2 * numpy.ones((2, 3))])),
            ConstantScheme(2))
        assert_raises(ValueError, next, Padding(stream).get_epoch_iterator()) 

def test_2d_sequences(self):
        stream = Batch(
            DataStream(
                IterableDataset([numpy.ones((3, 4)), 2 * numpy.ones((2, 4))])),
            ConstantScheme(2))
        it = Padding(stream).get_epoch_iterator()
        data, mask = next(it)
        assert data.shape == (2, 3, 4)
        assert (data[0, :, :] == 1).all()
        assert (data[1, :2, :] == 2).all()
        assert (mask == numpy.array([[1, 1, 1], [1, 1, 0]])).all() 

def test_1d_sequences(self):
        stream = Batch(
            DataStream(IterableDataset([[1], [2, 3], [], [4, 5, 6], [7]])),
            ConstantScheme(2))
        transformer = Padding(stream)
        assert_equal(transformer.sources, ("data", "data_mask"))
        assert_equal(list(transformer.get_epoch_iterator()),
                     [(numpy.array([[1, 0], [2, 3]]),
                       numpy.array([[1, 0], [1, 1]])),
                      (numpy.array([[0, 0, 0], [4, 5, 6]]),
                       numpy.array([[0, 0, 0], [1, 1, 1]])),
                      (numpy.array([[7]]), numpy.array([[1]]))])