Java Code Examples for org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator#reset()

The following examples show how to use org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator#reset() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ParallelInferenceTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test(timeout = 30000L)
    public void testInferenceSequential1() throws Exception {

        long count0 = 0;
        long count1 = 0;

        //We can't guarantee that on any particular run each thread will get data - it might randomly be assigned to
        // only one. Consequently: we'll run the test multiple times and ensure that in at least *some* of the test
        // runs both workers get some data.
        for (int i = 0; i < 20 && (count0 == 0 || count1 == 0); i++) {
            iterator = new MnistDataSetIterator(1, false, 12345);

            ParallelInference inf =
                    new ParallelInference.Builder(model).inferenceMode(InferenceMode.SEQUENTIAL).workers(2).build();

            try {

                log.info("Features shape: {}",
                        Arrays.toString(iterator.next().getFeatures().shapeInfoDataBuffer().asInt()));

                INDArray array1 = inf.output(iterator.next().getFeatures());
                INDArray array2 = inf.output(iterator.next().getFeatures());

                assertFalse(array1.isAttached());
                assertFalse(array2.isAttached());

                INDArray array3 = inf.output(iterator.next().getFeatures());
                assertFalse(array3.isAttached());

                iterator.reset();

                evalClassifcationSingleThread(inf, iterator);

                count0 = inf.getWorkerCounter(0);
                count1 = inf.getWorkerCounter(1);
//            System.out.println("Counts: " + count0 + ", " + count1);
            } finally {
                inf.shutdown();
            }
        }
        // both workers threads should have non-zero
        assertTrue(count0 > 0L);
        assertTrue(count1 > 0L);
    }

Example 2

Source File: ParallelInferenceTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test(timeout = 30000L)
    public void testInferenceSequential2() throws Exception {

        long count0 = 0;
        long count1 = 0;

        //We can't guarantee that on any particular run each thread will get data - it might randomly be assigned to
        // only one. Consequently: we'll run the test multiple times and ensure that in at least *some* of the test
        // runs both workers get some data.
        for (int i = 0; i < 20 && (count0 == 0 || count1 == 0); i++) {
            iterator = new MnistDataSetIterator(1, false, 12345);
            ParallelInference inf =
                    new ParallelInference.Builder(model).inferenceMode(InferenceMode.SEQUENTIAL).workers(2).build();

            try {

                log.info("Features shape: {}",
                        Arrays.toString(iterator.next().getFeatures().shapeInfoDataBuffer().asInt()));

                INDArray array1 = inf.output(iterator.next().getFeatures());
                INDArray array2 = inf.output(iterator.next().getFeatures());

                assertFalse(array1.isAttached());
                assertFalse(array2.isAttached());

                INDArray array3 = inf.output(iterator.next().getFeatures());
                assertFalse(array3.isAttached());

                iterator.reset();

                evalClassifcationMultipleThreads(inf, iterator, 10);

                // both workers threads should have non-zero
                count0 = inf.getWorkerCounter(0);
                count1 = inf.getWorkerCounter(1);
//            System.out.println("Counts: " + count0 + ", " + count1);
            } finally {
                inf.shutdown();
            }
        }
        assertTrue(count0 > 0L);
        assertTrue(count1 > 0L);
    }

Example 3

Source File: ParallelInferenceTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test(timeout = 30000L)
    public void testInferenceBatched1() throws Exception {
        long count0 = 0;
        long count1 = 0;

        //We can't guarantee that on any particular run each thread will get data - it might randomly be assigned to
        // only one. Consequently: we'll run the test multiple times and ensure that in at least *some* of the test
        // runs both workers get some data.
        for( int i=0; i<20 && (count0 == 0 || count1 == 0); i++ ) {
            ParallelInference inf = new ParallelInference.Builder(model).inferenceMode(InferenceMode.BATCHED).batchLimit(8)
                    .workers(2).build();
            try {

                iterator = new MnistDataSetIterator(1, false, 12345);


                log.info("Features shape: {}",
                        Arrays.toString(iterator.next().getFeatures().shapeInfoDataBuffer().asInt()));

                INDArray array1 = inf.output(iterator.next().getFeatures());
                INDArray array2 = inf.output(iterator.next().getFeatures());

                assertFalse(array1.isAttached());
                assertFalse(array2.isAttached());

                INDArray array3 = inf.output(iterator.next().getFeatures());
                assertFalse(array3.isAttached());

                iterator.reset();

                evalClassifcationMultipleThreads(inf, iterator, 10);

                // both workers threads should have non-zero
                count0 = inf.getWorkerCounter(0);
                count1 = inf.getWorkerCounter(1);
//            System.out.println("Counts: " + count0 + ", " + count1);
            } finally {
                inf.shutdown();
            }
        }
        assertTrue(count0 > 0L);
        assertTrue(count1 > 0L);
    }