Java Code Examples for org.deeplearning4j.nn.conf.WorkspaceMode#NONE

public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
                     double realMin, double initialMomentum, double finalMomentum, double momentum,
                     int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
                     double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
                     double minGain,int vpTreeWorkers) {
    this(numDimensions, simiarlityFunction, theta, invert, maxIter, realMin, initialMomentum, finalMomentum,
            momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, learningRate,
            useAdaGrad, perplexity, TrainingListener, minGain, vpTreeWorkers, WorkspaceMode.NONE, null);
}
 

public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter,
                     double realMin, double initialMomentum, double finalMomentum, double momentum,
                     int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance,
                     double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener,
                     double minGain,int vpTreeWorkers, WorkspaceMode workspaceMode, INDArray staticInput) {
    this.maxIter = maxIter;
    this.realMin = realMin;
    this.initialMomentum = initialMomentum;
    this.finalMomentum = finalMomentum;
    this.momentum = momentum;
    this.normalize = normalize;
    this.useAdaGrad = useAdaGrad;
    this.stopLyingIteration = stopLyingIteration;
    this.learningRate = learningRate;
    this.switchMomentumIteration = switchMomentumIteration;
    this.tolerance = tolerance;
    this.perplexity = perplexity;
    this.minGain = minGain;
    this.numDimensions = numDimensions;
    this.simiarlityFunction = simiarlityFunction;
    this.theta = theta;
    this.trainingListener = TrainingListener;
    this.invert = invert;
    this.vpTreeWorkers = vpTreeWorkers;
    this.workspaceMode = workspaceMode;
    if(this.workspaceMode == null)
        this.workspaceMode = WorkspaceMode.NONE;
    initializer = (staticInput != null) ? new Initializer(staticInput) : new Initializer();
}
 

@Test
public void testSimple() throws Exception {
    //Simple sanity check

    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(250, 200);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
}
 

@Test
public void testPerformance() throws Exception {

    StopWatch watch = new StopWatch();
    watch.start();
    for( int test=0; test <=1; test++){
        boolean syntheticData = test == 1;
        WorkspaceMode wsm = test == 0 ? WorkspaceMode.NONE : WorkspaceMode.ENABLED;
        log.info("Starting test: WSM={}, syntheticData={}", wsm, syntheticData);

        //STEP 1: Initialization
        int iterations = 50;
        //create an n-dimensional array of doubles
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);
        List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

        //STEP 2: Turn text input into a list of words
        INDArray weights;
        if(syntheticData){
            weights = Nd4j.rand(DataType.FLOAT, 250, 20);
        } else {
            log.info("Load & Vectorize data....");
            File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile();   //Open the file
            //Get the data of all unique word vectors
            Pair<InMemoryLookupTable, VocabCache> vectors = WordVectorSerializer.loadTxt(wordFile);
            VocabCache cache = vectors.getSecond();
            weights = vectors.getFirst().getSyn0();    //seperate weights of unique words into their own list

            for (int i = 0; i < cache.numWords(); i++)   //seperate strings of words into their own list
                cacheList.add(cache.wordAtIndex(i));
        }

        //STEP 3: build a dual-tree tsne to use later
        log.info("Build model....");
        BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                .setMaxIter(iterations)
                .theta(0.5)
                .normalize(false)
                .learningRate(500)
                .useAdaGrad(false)
                .workspaceMode(wsm)
                .build();


        //STEP 4: establish the tsne values and save them to a file
        log.info("Store TSNE Coordinates for Plotting....");
        File outDir = testDir.newFolder();
        tsne.fit(weights);
        tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
    }
    watch.stop();
    System.out.println("Elapsed time : " + watch);
}
 

@Ignore
@Test
public void testTSNEPerformance() throws Exception {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            //STEP 1: Initialization
            int iterations = 50;
            //create an n-dimensional array of doubles
            Nd4j.setDataType(DataType.DOUBLE);
            List<String> cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words

            //STEP 2: Turn text input into a list of words
            INDArray weights = Nd4j.rand(10000,300);

            StopWatch watch = new StopWatch();
            watch.start();
            //STEP 3: build a dual-tree tsne to use later
            log.info("Build model....");
            BarnesHutTsne tsne = new BarnesHutTsne.Builder()
                    .setMaxIter(iterations)
                    .theta(0.5)
                    .normalize(false)
                    .learningRate(500)
                    .useAdaGrad(false)
                    .workspaceMode(wsm)
                    .build();

            watch.stop();
            System.out.println("Elapsed time for construction: " + watch);

            //STEP 4: establish the tsne values and save them to a file
            log.info("Store TSNE Coordinates for Plotting....");
            File outDir = testDir.newFolder();

            watch.reset();
            watch.start();
            tsne.fit(weights);
            watch.stop();
            System.out.println("Elapsed time for fit: " + watch);
            tsne.saveAsFile(cacheList, new File(outDir, "out.txt").getAbsolutePath());
        }
}
 

@Test
public void testTimeDistributed(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {

        MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build())
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new LSTM.Builder().nIn(3).nOut(3).dataFormat(rnnDataFormat).build())
                .layer(new TimeDistributed(new DenseLayer.Builder().nIn(3).nOut(3).activation(Activation.TANH).build(), rnnDataFormat))
                .layer(new RnnOutputLayer.Builder().nIn(3).nOut(3).activation(Activation.SOFTMAX).dataFormat(rnnDataFormat)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(3, rnnDataFormat))
                .build();

        MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net1.init();
        net2.init();

        for( int mb : new int[]{1, 5}) {
            for(char inLabelOrder : new char[]{'c', 'f'}) {
                INDArray in = Nd4j.rand(DataType.FLOAT, mb, 3, 5).dup(inLabelOrder);
                if (rnnDataFormat == RNNFormat.NWC){
                    in = in.permute(0, 2, 1);
                }
                INDArray out1 = net1.output(in);
                INDArray out2 = net2.output(in);
                assertEquals(out1, out2);

                INDArray labels ;
                if (rnnDataFormat == RNNFormat.NCW) {
                    labels = TestUtils.randomOneHotTimeSeries(mb, 3, 5).dup(inLabelOrder);
                }else{
                    labels = TestUtils.randomOneHotTimeSeries(mb, 5, 3).dup(inLabelOrder);
                }



                DataSet ds = new DataSet(in, labels);
                net1.fit(ds);
                net2.fit(ds);

                assertEquals(net1.params(), net2.params());

                MultiLayerNetwork net3 = TestUtils.testModelSerialization(net2);
                out2 = net2.output(in);
                INDArray out3 = net3.output(in);

                assertEquals(out2, out3);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForward() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,
                    Activation.HARDTANH,
                    Activation.RELU
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + ", workspace = " + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .inferenceWorkspaceMode(wsm)
                        .trainingWorkspaceMode(wsm)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .activation(a)
                                .build())
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                net.params().assign(net2.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);

                //Sanity check on different minibatch sizes:
                INDArray newIn = Nd4j.vstack(in, in);
                INDArray outMbsd = net.output(newIn);
                INDArray outMb = net2.output(newIn);
                assertEquals(outMb, outMbsd);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseForwardMultiLayer() {
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        for (int minibatch : new int[]{5, 1}) {
            int nIn = 3;
            int nOut = 4;

            Activation[] afns = new Activation[]{
                    Activation.TANH,
                    Activation.SIGMOID,
                    Activation.ELU,
                    Activation.IDENTITY,
                    Activation.SOFTPLUS,
                    Activation.SOFTSIGN,
                    Activation.CUBE,    //https://github.com/deeplearning4j/nd4j/issues/2426
                    Activation.HARDTANH,
                    Activation.RELU      //JVM crash
            };

            for (Activation a : afns) {
                log.info("Starting test - " + a + " - workspace=" + wsm);
                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .list()
                        .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new SameDiffDense.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .weightInit(WeightInit.XAVIER)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                assertNotNull(net.paramTable());

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .seed(12345)
                        .weightInit(WeightInit.XAVIER)
                        .list()
                        .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                        .layer(new DenseLayer.Builder().activation(a).nIn(nOut).nOut(nOut).build())
                        .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut)
                                .activation(a).build())
                        .validateOutputLayerConfig(false)
                        .build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                assertEquals(net2.params(), net.params());

                //Check params:
                assertEquals(net2.params(), net.params());
                Map<String, INDArray> params1 = net.paramTable();
                Map<String, INDArray> params2 = net2.paramTable();
                assertEquals(params2, params1);

                INDArray in = Nd4j.rand(minibatch, nIn);
                INDArray out = net.output(in);
                INDArray outExp = net2.output(in);

                assertEquals(outExp, out);

                //Also check serialization:
                MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);
                INDArray outLoaded = netLoaded.output(in);

                assertEquals(outExp, outLoaded);


                //Sanity check different minibatch sizes
                in = Nd4j.rand(2 * minibatch, nIn);
                out = net.output(in);
                outExp = net2.output(in);
                assertEquals(outExp, out);
            }
        }
    }
}
 

@Test
public void testSameDiffDenseTraining() {
    Nd4j.getRandom().setSeed(12345);

    int nIn = 4;
    int nOut = 3;

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .updater(new Adam(0.1))
                .list()
                .layer(new SameDiffDense.Builder().nIn(nIn).nOut(5).activation(Activation.TANH).build())
                .layer(new SameDiffDense.Builder().nIn(5).nOut(5).activation(Activation.TANH).build())
                .layer(new OutputLayer.Builder().nIn(5).nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .build();

        MultiLayerNetwork netSD = new MultiLayerNetwork(conf);
        netSD.init();

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(0.1))
                .list()
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(nIn).nOut(5).build())
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(5).build())
                .layer(new OutputLayer.Builder().nIn(5).nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .build();

        MultiLayerNetwork netStandard = new MultiLayerNetwork(conf2);
        netStandard.init();

        netSD.params().assign(netStandard.params());

        //Check params:
        assertEquals(netStandard.params(), netSD.params());
        assertEquals(netStandard.paramTable(), netSD.paramTable());

        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        DataSet ds = iter.next();

        INDArray outSD = netSD.output(ds.getFeatures());
        INDArray outStd = netStandard.output(ds.getFeatures());

        assertEquals(outStd, outSD);

        for (int i = 0; i < 3; i++) {
            netSD.fit(ds);
            netStandard.fit(ds);
            String s = String.valueOf(i);
            assertEquals(s, netStandard.getFlattenedGradients(), netSD.getFlattenedGradients());
            assertEquals(s, netStandard.params(), netSD.params());
            assertEquals(s, netStandard.getUpdater().getStateViewArray(), netSD.getUpdater().getStateViewArray());
        }

        //Sanity check on different minibatch sizes:
        INDArray newIn = Nd4j.vstack(ds.getFeatures(), ds.getFeatures());
        INDArray outMbsd = netSD.output(newIn);
        INDArray outMb = netStandard.output(newIn);
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testSameDiffLamdaLayerBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);


        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in")
                .addLayer("1", new SameDiffSimpleLambdaLayer(), "0")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "1")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in")
                .addVertex("1", new ShiftVertex(1.0), "0")
                .addVertex("2", new ScaleVertex(2.0), "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "2")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        DataSet ds = new DataSet(in, labels);

        INDArray outLambda = lambda.outputSingle(in);
        INDArray outStd = std.outputSingle(in);

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(ds);
        double scoreStd = std.score(ds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(ds);
            std.fit(ds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.outputSingle(in);
        outStd = std.outputSingle(in);

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn = Nd4j.vstack(in, in);
        INDArray outMbsd = lambda.output(newIn)[0];
        INDArray outMb = std.output(newIn)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void testSameDiffLamdaVertexBasic(){
    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) {
        log.info("--- Workspace Mode: {} ---", wsm);

        Nd4j.getRandom().setSeed(12345);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("lambda", new SameDiffSimpleLambdaVertex(), "0", "1")
                .addLayer("2", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "lambda")
                .setOutputs("2")
                .build();

        //Equavalent, not using SameDiff Lambda:
        ComputationGraphConfiguration confStd = new NeuralNetConfiguration.Builder()
                .trainingWorkspaceMode(wsm)
                .inferenceWorkspaceMode(wsm)
                .dataType(DataType.DOUBLE)
                .seed(12345)
                .updater(new Adam(0.01))
                .graphBuilder()
                .addInputs("in1", "in2")
                .addLayer("0", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in1")
                .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build(), "in2")
                .addVertex("elementwise", new ElementWiseVertex(ElementWiseVertex.Op.Product), "0", "1")
                .addLayer("3", new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "elementwise")
                .setOutputs("3")
                .build();

        ComputationGraph lambda = new ComputationGraph(conf);
        lambda.init();

        ComputationGraph std = new ComputationGraph(confStd);
        std.init();

        lambda.setParams(std.params());

        INDArray in1 = Nd4j.rand(3, 5);
        INDArray in2 = Nd4j.rand(3, 5);
        INDArray labels = TestUtils.randomOneHot(3, 5);
        MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{in1, in2}, new INDArray[]{labels});

        INDArray outLambda = lambda.output(in1, in2)[0];
        INDArray outStd = std.output(in1, in2)[0];

        assertEquals(outLambda, outStd);

        double scoreLambda = lambda.score(mds);
        double scoreStd = std.score(mds);

        assertEquals(scoreStd, scoreLambda, 1e-6);

        for (int i = 0; i < 3; i++) {
            lambda.fit(mds);
            std.fit(mds);

            String s = String.valueOf(i);
            assertEquals(s, std.params(), lambda.params());
            assertEquals(s, std.getFlattenedGradients(), lambda.getFlattenedGradients());
        }

        ComputationGraph loaded = TestUtils.testModelSerialization(lambda);
        outLambda = loaded.output(in1, in2)[0];
        outStd = std.output(in1, in2)[0];

        assertEquals(outStd, outLambda);

        //Sanity check on different minibatch sizes:
        INDArray newIn1 = Nd4j.vstack(in1, in1);
        INDArray newIn2 = Nd4j.vstack(in2, in2);
        INDArray outMbsd = lambda.output(newIn1, newIn2)[0];
        INDArray outMb = std.output(newIn1, newIn2)[0];
        assertEquals(outMb, outMbsd);
    }
}
 

@Test
public void compareImplementations() throws Exception {

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .updater(new Adam(1e-3))
                .weightInit(WeightInit.XAVIER)
                .inferenceWorkspaceMode(wsm)
                .trainingWorkspaceMode(wsm)
                .list()
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(784)
                        .nOut(32)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID))
                        .build())
                .layer(new VariationalAutoencoder.Builder()
                        .nIn(32)
                        .nOut(8)
                        .encoderLayerSizes(16)
                        .decoderLayerSizes(16)
                        .activation(Activation.TANH)
                        .pzxActivationFunction(Activation.SIGMOID)
                        .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
                        .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf2);
        net.init();

        ComputationGraph cg = net.toComputationGraph();
        cg.getConfiguration().setInferenceWorkspaceMode(wsm);
        cg.getConfiguration().setTrainingWorkspaceMode(wsm);
        DataSetIterator ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        net.pretrainLayer(0, ds);

        ds = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(1, true, 12345), 1);
        Nd4j.getRandom().setSeed(12345);
        cg.pretrainLayer("0", ds);

        assertEquals(net.params(), cg.params());
    }
}
 

public static void validateLayers(MultiLayerNetwork net, List<Class<?>> classesToTest, boolean testAllCudnnPresent, int[] fShape, int[] lShape, double maxRE, double minAbsErr) {

        for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {

            net.getLayerWiseConfigurations().setTrainingWorkspaceMode(wsm);
            net.getLayerWiseConfigurations().setInferenceWorkspaceMode(wsm);

            Nd4j.getRandom().setSeed(12345);
            INDArray features = Nd4j.rand(fShape);
            INDArray labels = Nd4j.rand(lShape);
            labels = Nd4j.exec(new IsMax(labels, 1))[0].castTo(features.dataType());

            List<CuDNNValidationUtil.TestCase> testCaseList = new ArrayList<>();

            List<DataSet> dataSets = new ArrayList<>();
            for (int i = 0; i < 6; i++) {
                INDArray f = Nd4j.rand(fShape);
                INDArray l = Nd4j.rand(lShape);
                l = Nd4j.exec(new IsMax(l, 1))[0].castTo(features.dataType());
                dataSets.add(new DataSet(f, l));
            }
            DataSetIterator iter = new ExistingDataSetIterator(dataSets);

            for (Class<?> c : classesToTest) {
                String name = "WS=" + wsm + ", testCudnnFor=" + c.getSimpleName();
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName(name)
                        .allowCudnnHelpersForClasses(Collections.<Class<?>>singletonList(c))
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .testTraining(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            if(testAllCudnnPresent) {
                testCaseList.add(CuDNNValidationUtil.TestCase.builder()
                        .testName("WS=" + wsm + ", ALL CLASSES")
                        .allowCudnnHelpersForClasses(classesToTest)
                        .testForward(true)
                        .testScore(true)
                        .testBackward(true)
                        .trainFirst(false)
                        .features(features)
                        .labels(labels)
                        .data(iter)
                        .maxRE(maxRE)
                        .minAbsErr(minAbsErr)
                        .build());
            }

            for (CuDNNValidationUtil.TestCase tc : testCaseList) {
                log.info("Running test: " + tc.getTestName());
                CuDNNValidationUtil.validateMLN(net, tc);
            }
        }
    }
 

@Test
public void testEqualUuid1() {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())
            .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(2, new ConvolutionLayer.Builder(5, 5)
                    //Note that nIn needed be specified in later layers
                    .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build())
            .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build())
            .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    // ParallelWrapper will take care of load balancing between GPUs.
    ParallelWrapper wrapper = new ParallelWrapper.Builder(model)
            // DataSets prefetching options. Set this value with respect to number of actual devices
            .prefetchBuffer(24)

            // set number of workers equal or higher then number of available devices. x1-x2 are good values to start with
            .workers(2)

            // rare averaging improves performance, but might reduce model accuracy
            .averagingFrequency(3)

            // if set to TRUE, on every averaging model score will be reported
            .reportScoreAfterAveraging(true)

            // optinal parameter, set to false ONLY if your system has support P2P memory access across PCIe (hint: AWS do not support P2P)
            .build();

    val trainer = new SymmetricTrainer(model, "alpha", 3, WorkspaceMode.NONE, wrapper, true);

    assertEquals("alpha_thread_3", trainer.getUuid());
}