Java Code Examples for org.deeplearning4j.nn.conf.WorkspaceMode#ENABLED

public static PretrainedNNFilter getPretrainedNNFilterByName(final String name, final int layer,
                                                             final long[] shape) {
    // Thanks to a pointless API requirement, the zoo models require int[] shapes
    // while dl4j uses long[] at any other place
    final int[] intShape = Arrays.stream(shape).mapToInt(i -> (int) i).toArray();

    switch (name) {
        case "AlexNet":
            return new PretrainedNNFilter(new AlexNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "LeNet":
            return new PretrainedNNFilter(new LeNet(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "VGG19":
            return new PretrainedNNFilter(new VGG19(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, name);
        case "ResNet50":
            return new PretrainedNNFilter(new ResNet50(42, intShape, 10, WeightInit.DISTRIBUTION,
                    new Nesterovs(1e-2, 0.9), CacheMode.NONE, WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer,
                    shape, name);
        default:
            return new PretrainedNNFilter(new VGG16(42, intShape, 10, new Nesterovs(1e-2, 0.9), CacheMode.NONE,
                    WorkspaceMode.ENABLED, AlgoMode.PREFER_FASTEST), layer, shape, "VGG16");
    }
}
 

@Test
public void testSimple() throws Exception {
    //Simple sanity check

    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(250, 200);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
}
 

@Test
public void testPerformance() throws Exception {

    StopWatch watch = new StopWatch();
    watch.start();
    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(DataType.FLOAT, 250, 20);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
    watch.stop();
    System.out.println("Elapsed time : " + watch);
}
 

@Ignore
@Test
public void testTSNEPerformance() throws Exception {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            //STEP 1: Initialization
            int iterations = 50;
            //create an n-dimensional array of doubles
            Nd4j.setDataType(DataType.DOUBLE);
            List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

            //STEP 2: Turn text input into a list of words
            INDArray weights = Nd4j.rand(10000,300);

            StopWatch watch = new StopWatch();
            watch.start();
            //STEP 3: build a dual-tree tsne to use later
            log.info("Build model....");
            BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                    .setMaxIter(iterations)
                    .theta(0.5)
                    .normalize(false)
                    .learningRate(500)
                    .useAdaGrad(false)
                    .workspaceMode(wsm)
                    .build();

            watch.stop();
            System.out.println("Elapsed time for construction: " + watch);

            //STEP 4: establish the tsne values and save them to a file
            log.info("Store TSNE Coordinates for Plotting....");
            File outDir = testDir.newFolder();

            watch.reset();
            watch.start();
            tsne.fit(weights);
            watch.stop();
            System.out.println("Elapsed time for fit: " + watch);
            tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
        }
}
 

@Test
public void testTimeDistributed(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {

        MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build())
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new TimeDistributed(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build(), rnnDataFormat))
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net1.init();
        net2.init();

        for( int mb : new int[]{1, 5}) {
            for(char inLabelOrder : new char[]{'c', 'f'}) {
                INDArray in = Nd4j.rand(DataType.FLOAT, mb, 3, 5).dup(inLabelOrder);
                if (rnnDataFormat == RNNFormat.NWC){
                    in = in.permute(0, 2, 1);
                }
                INDArray out1 = net1.output(in);
                INDArray out2 = net2.output(in);
                assertEquals(out1, out2);

                INDArray labels ;
                if (rnnDataFormat == RNNFormat.NCW) {
                    labels = TestUtils.randomOneHotTimeSeries(mb, 3, 5).dup(inLabelOrder);
                }else{
                    labels = TestUtils.randomOneHotTimeSeries(mb, 5, 3).dup(inLabelOrder);
                }



                DataSet ds = new DataSet(in, labels);
                net1.fit(ds);
                net2.fit(ds);

                assertEquals(net1.params(), net2.params());

                MultiLayerNetwork net3 = TestUtils.testModelSerialization(net2);
                out2 = net2.output(in);
                INDArray out3 = net3.output(in);

                assertEquals(out2, out3);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForward() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,
                    Activation.HARDTANH,
                    Activation.RELU
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + ", workspace = " + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .inferenceWorkspaceMode(wsm)
                        .trainingWorkspaceMode(wsm)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .activation(a)
                                .build())
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                net.params().assign(net2.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);

                //Sanity check on different minibatch sizes:
                INDArray newIn = Nd4j.vstack(in, in);
                INDArray outMbsd = net.output(newIn);
                INDArray outMb = net2.output(newIn);
                assertEquals(outMb, outMbsd);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForwardMultiLayer() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,    //https://github.com/deeplearning4j/nd4j/issues/2426
                    Activation.HARDTANH,
                    Activation.RELU      //JVM crash
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + " - workspace=" + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new SameDiffDense.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .weightInit(WeightInit.XAVIER)
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .layer(new DenseLayer.Builder().activation(a).nIn(nOut).nOut(nOut).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                assertEquals(net2.params(), net.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);


                //Sanity check different minibatch sizes
                in = Nd4j.rand(2 * minibatch, nIn);
                out = net.output(in);
                outExp = net2.output(in);
                assertEquals(outExp, out);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseTraining() {
    Nd4j.getRandom().setSeed(12345);

    int nIn = 4;
    int nOut = 3;

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .updater(new Adam(0.1))
                .list()
                .layer(new SameDiffDense.Builder().nIn(nIn).nOut(5).activation(Activation.TANH).build())
                .layer(new SameDiffDense.Builder().nIn(5).nOut(5).activation(Activation.TANH).build())
                .layer(new OutputLayer.Builder().nIn(5).nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .build();

        MultiLayerNetwork netSD = new MultiLayerNetwork(conf);
        netSD.init();

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(nIn).nOut(5).build())
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(5).build())
                .layer(new OutputLayer.Builder().nIn(5).nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .build();

        MultiLayerNetwork netStandard = new MultiLayerNetwork(conf2);
        netStandard.init();

        netSD.params().assign(netStandard.params());

        //Check params:
        assertEquals(netStandard.params(), netSD.params());
        assertEquals(netStandard.paramTable(), netSD.paramTable());

        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        DataSet ds = iter.next();

        INDArray outSD = netSD.output(ds.getFeatures());
        INDArray outStd = netStandard.output(ds.getFeatures());

        assertEquals(outStd, outSD);

        for (int i = 0; i < 3; i++) {
            netSD.fit(ds);
            netStandard.fit(ds);
            String s = String.valueOf(i);
            assertEquals(s, netStandard.getFlattenedGradients(), netSD.getFlattenedGradients());
            assertEquals(s, netStandard.params(), netSD.params());
            assertEquals(s, netStandard.getUpdater().getStateViewArray(), netSD.getUpdater().getStateViewArray());
        }

        //Sanity check on different minibatch sizes:
        INDArray newIn = Nd4j.vstack(ds.getFeatures(), ds.getFeatures());
        INDArray outMbsd = netSD.output(newIn);
        INDArray outMb = netStandard.output(newIn);
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testSameDiffLamdaLayerBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);


        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in")
                .addLayer("1", new SameDiffSimpleLambdaLayer(), "0")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "1")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in")
                .addVertex("1", new ShiftVertex(1.0), "0")
                .addVertex("2", new ScaleVertex(2.0), "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "2")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        DataSet ds = new DataSet(in, labels);

        INDArray outLambda = lambda.outputSingle(in);
        INDArray outStd = std.outputSingle(in);

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(ds);
        double scoreStd = std.score(ds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(ds);
            std.fit(ds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.outputSingle(in);
        outStd = std.outputSingle(in);

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn = Nd4j.vstack(in, in);
        INDArray outMbsd = lambda.output(newIn)[0];
        INDArray outMb = std.output(newIn)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testSameDiffLamdaVertexBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);

        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("lambda", new SameDiffSimpleLambdaVertex(), "0", "1")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lambda")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("elementwise", new ElementWiseVertex(ElementWiseVertex.Op.Product), "0", "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "elementwise")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in1 = Nd4j.rand(3, 5);
        INDArray in2 = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{in1, in2}, new INDArray[]{labels});

        INDArray outLambda = lambda.output(in1, in2)[0];
        INDArray outStd = std.output(in1, in2)[0];

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(mds);
        double scoreStd = std.score(mds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(mds);
            std.fit(mds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.output(in1, in2)[0];
        outStd = std.output(in1, in2)[0];

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn1 = Nd4j.vstack(in1, in1);
        INDArray newIn2 = Nd4j.vstack(in2, in2);
        INDArray outMbsd = lambda.output(newIn1, newIn2)[0];
        INDArray outMb = std.output(newIn1, newIn2)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void compareImplementations() throws Exception {

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(1e-3))
                .weightInit(WeightInit.XAVIER)
                .inferenceWorkspaceMode(wsm)
                .trainingWorkspaceMode(wsm)
                .list()
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(784)
                        .nOut(32)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID))
                        .build())
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(32)
                        .nOut(8)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
                        .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf2);
        net.init();

        ComputationGraph cg = net.toComputationGraph();
        cg.getConfiguration().setInferenceWorkspaceMode(wsm);
        cg.getConfiguration().setTrainingWorkspaceMode(wsm);
        DataSetIterator ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        net.pretrainLayer(0, ds);

        ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        cg.pretrainLayer("0", ds);

        assertEquals(net.params(), cg.params());
    }
}
 

public static void validateLayers(MultiLayerNetwork net, List<Class<?>> classesToTest, boolean testAllCudnnPresent, int[] fShape, int[] lShape, double maxRE, double minAbsErr) {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            net.getLayerWiseConfigurations().setTrainingWorkspaceMode(wsm);
            net.getLayerWiseConfigurations().setInferenceWorkspaceMode(wsm);

            Nd4j.getRandom().setSeed(12345);
            INDArray features = Nd4j.rand(fShape);
            INDArray labels = Nd4j.rand(lShape);
            labels = Nd4j.exec(new IsMax(labels, 1))[0].castTo(features.dataType());

            List<CuDNNValidationUtil.TestCase> testCaseList = new ArrayList<>();

            List<DataSet> dataSets = new ArrayList<>();
            for (int i = 0; i < 6; i++) {
                INDArray f = Nd4j.rand(fShape);
                INDArray l = Nd4j.rand(lShape);
                l = Nd4j.exec(new IsMax(l, 1))[0].castTo(features.dataType());
                dataSets.add(new DataSet(f, l));
            }
            DataSetIterator iter = new ExistingDataSetIterator(dataSets);

            for (Class<?> c : classesToTest) {
                String name = "WS=" + wsm + ", testCudnnFor=" + c.getSimpleName();
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName(name)
                        .allowCudnnHelpersForClasses(Collections.<Class<?>>singletonList(c))
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .testTraining(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            if(testAllCudnnPresent) {
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName("WS=" + wsm + ", ALL CLASSES")
                        .allowCudnnHelpersForClasses(classesToTest)
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            for (CuDNNValidationUtil.TestCase tc : testCaseList) {
                log.info("Running test: " + tc.getTestName());
                CuDNNValidationUtil.validateMLN(net, tc);
            }
        }
    }