Java Code Examples for org.deeplearning4j.nn.multilayer.MultiLayerNetwork#output()

private void importEmbeddingConv1D(String modelPath) throws Exception {
    MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false);

    int nIn = 4;
    int nOut = 6;
    int outputDim = 5;
    int inputLength = 10;
    int kernel = 3;
    int mb = 42;

    INDArray embeddingWeight = model.getLayer(0).getParam("W");
    val embeddingWeightShape = embeddingWeight.shape();
    assertEquals(nIn, embeddingWeightShape[0]);
    assertEquals(outputDim, embeddingWeightShape[1]);

    INDArray inEmbedding = Nd4j.zeros(mb, inputLength);
    INDArray output = model.output(inEmbedding);
    assertArrayEquals(new long[]{mb, inputLength - kernel + 1, nOut}, output.shape());      //NWC
    logSuccess(modelPath);
}
 

@Test
public void testConvCacheModeSimple(){

    MultiLayerConfiguration conf1 = getConf(CacheMode.NONE);
    MultiLayerConfiguration conf2 = getConf(CacheMode.DEVICE);

    MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
    net1.init();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    INDArray in = Nd4j.rand(3, 28*28);
    INDArray labels = TestUtils.randomOneHot(3, 10);

    INDArray out1 = net1.output(in);
    INDArray out2 = net2.output(in);
    assertEquals(out1, out2);

    assertEquals(net1.params(), net2.params());
    net1.fit(in, labels);
    net2.fit(in, labels);
    assertEquals(net1.params(), net2.params());
}
 

@Test
public void testLSTMCacheModeSimple(){

    for(boolean graves : new boolean[]{true, false}) {

        MultiLayerConfiguration conf1 = getConfLSTM(CacheMode.NONE, graves);
        MultiLayerConfiguration conf2 = getConfLSTM(CacheMode.DEVICE, graves);

        MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
        net1.init();
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net2.init();

        INDArray in = Nd4j.rand(new int[]{3, 3, 10});
        INDArray labels = TestUtils.randomOneHotTimeSeries(3, 10, 10);

        INDArray out1 = net1.output(in);
        INDArray out2 = net2.output(in);
        assertEquals(out1, out2);

        assertEquals(net1.params(), net2.params());
        net1.fit(in, labels);
        net2.fit(in, labels);
        assertEquals(net1.params(), net2.params());
    }
}
 

@Test
public void testSequentialLambdaLayerImport() throws Exception {
    KerasLossUtils.registerCustomLoss("logcosh", new LogCosh());

    String modelPath = "modelimport/keras/examples/custom_loss.h5";

    try(InputStream is = Resources.asStream(modelPath)) {
        File modelFile = testDir.newFile("tempModel" + System.currentTimeMillis() + ".h5");
        Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
        MultiLayerNetwork model = new KerasSequentialModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath())
                .enforceTrainingConfig(true).buildSequential().getMultiLayerNetwork();

        System.out.println(model.summary());
        INDArray input = Nd4j.create(new int[]{10, 3});

        model.output(input);
    } finally {
        KerasLossUtils.clearCustomLoss();
    }
}
 

@Test
public void testTransferLearningSubsequent() {
    final INDArray input = Nd4j.create(6,6,6,6);
    final MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
            .weightInit(new ConstantDistribution(666))
            .list()
            .setInputType(InputType.inferInputTypes(input)[0])
            .layer(new Convolution2D.Builder(3, 3).nOut(10).build())
            .layer(new Convolution2D.Builder(1, 1).nOut(3).build())
            .layer(new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE)
                    .build()).build());
    net.init();

    MultiLayerNetwork newGraph = new TransferLearning
            .Builder(net)
            .fineTuneConfiguration(new FineTuneConfiguration.Builder().build())
            .nOutReplace(0, 7, new ConstantDistribution(333))
            .nOutReplace(1, 3, new ConstantDistribution(111))
            .removeLayersFromOutput(1)
            .addLayer(new OutputLayer.Builder()
                    .nIn(48).nOut(2)
                    .lossFunction(LossFunctions.LossFunction.MSE)
                    .build())
            .setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(4,4,3))
            .build();
    newGraph.init();

    assertEquals("Incorrect # inputs", 7, newGraph.layerInputSize(1));

    newGraph.output(input);
}
 

private void importSimpleSpaceToDepth(String modelPath) throws Exception {
    KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class);
    MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false);

    INDArray input = Nd4j.zeros(10, 6, 6, 4);
    INDArray output = model.output(input);
    assertArrayEquals(new long[]{10, 3, 3, 16}, output.shape());
    logSuccess(modelPath);
}
 

@Test
public void test5982() throws Exception {
    File jsonFile = Resources.asFile("modelimport/keras/configs/bidirectional_last_timeStep.json");
    val modelGraphConf = KerasModelImport.importKerasSequentialConfiguration(jsonFile.getAbsolutePath());
    MultiLayerNetwork model = new MultiLayerNetwork(modelGraphConf);

    INDArray features = Nd4j.create(new double[]{1, 3, 1, 2, 2, 1, 82, 2, 10,1, 3, 1, 2, 1, 82, 3, 1, 10, 1, 2, 1, 3,
            1, 10, 82, 2, 1, 1, 10, 82, 2, 3, 1, 2, 1, 10, 1, 2, 3, 82, 2, 1, 10, 3, 82, 1, 2, 1, 10, 1}, new int[]{1,1,50});

    model.init();
    INDArray out = model.output(features);
    assertArrayEquals(new long[]{1,14}, out.shape());
}
 

public static void main(String[] args){

        //Generate the training data
        DataSetIterator iterator = getTrainingData(batchSize,rng);

        //Create the network
        int numInput = 2;
        int numOutputs = 1;
        MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
                .seed(seed)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .weightInit(WeightInit.XAVIER)
                .updater(new Sgd(learningRate))
                .list()
                .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE)
                        .activation(Activation.IDENTITY)
                        .nIn(numInput).nOut(numOutputs).build())
                .pretrain(false).backprop(true).build()
        );
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        for( int i=0; i<nEpochs; i++ ){
            iterator.reset();
            net.fit(iterator);
        }

        final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 });
        INDArray out = net.output(input, false);
        System.out.println(out);

    }
 

@Test
public void testEmbeddingLongerSequencesForwardPass() {

    int nClassesIn = 10;
    int inputLength = 6;
    int embeddingDim = 5;
    int nOut = 4;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
            .layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength)
                    .hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build())
            .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build())
            .build();


    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    int batchSize = 3;

    INDArray inEmbedding = Nd4j.create(batchSize, inputLength);

    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
    }

    INDArray output = net.output(inEmbedding);

    assertArrayEquals(new long[]{batchSize, nOut, inputLength}, output.shape());
}
 

@Test
public void test1dForward(){
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4)
            .updater(new Nesterovs(0.9)).dropOut(0.5)
            .list()
            .layer(new LocallyConnected1D.Builder().kernelSize(4).nIn(3)
                    .stride(1).nOut(16).dropOut(0.5)
                    .convolutionMode(ConvolutionMode.Strict)
                    .setInputSize(28)
                    .activation(Activation.RELU).weightInit(
                            WeightInit.XAVIER)
                    .build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer
                    .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.recurrent(3,  8));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    INDArray input = Nd4j.ones(10, 3, 8);
    INDArray output = network.output(input, false);;
    for (int i = 0; i < 100; i++) { // TODO: this falls flat for 1000 iterations on my machine
        output = network.output(input, false);
    }

    assertArrayEquals(new long[] {(8 - 4 + 1) * 10, 10}, output.shape());
    network.fit(input, output);

}
 

@Test
    public void testLabelProbabilities() throws Exception {
        Nd4j.getRandom().setSeed(42);
        DataSetIterator dataSetIterator = getNormalizedIterator();
        MultiLayerNetwork network = getSingleLayer();
        DataSet next = dataSetIterator.next();
        DataSet filtered = next.filterBy(new int[]{0, 1});
        for (int i = 0; i < 10; i++) {
            network.setEpochCount(i);
            network.getLayerWiseConfigurations().setEpochCount(i);
            network.fit(filtered);
        }

        DataSet anomalies = next.filterBy(new int[] {2});
        INDArray output = network.output(anomalies.getFeatures());
        INDArray normalOutput = network.output(anomalies.getFeatures(),false);
        assertEquals(output.lt(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(),
                normalOutput.eq(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(),1e-1);

//        System.out.println("Labels " + anomalies.getLabels());
//        System.out.println("Anomaly output " + normalOutput);
//        System.out.println(output);

        INDArray normalProbs = network.output(filtered.getFeatures());
        INDArray outputForNormalSamples = network.output(filtered.getFeatures(),false);
        System.out.println("Normal probabilities " + normalProbs);
        System.out.println("Normal raw output " + outputForNormalSamples);

        File tmpFile = new File(testDir.getRoot(),"tmp-file-" + UUID.randomUUID().toString());
        ModelSerializer.writeModel(network,tmpFile,true);
        tmpFile.deleteOnExit();

        MultiLayerNetwork multiLayerNetwork = ModelSerializer.restoreMultiLayerNetwork(tmpFile);
        assertEquals(network.params(),multiLayerNetwork.params());
        assertEquals(network.numParams(),multiLayerNetwork.numParams());

    }
 

@Test
public void testBatchNormRecurrentCnn1d() {
    //Simple sanity check on CNN1D and RNN layers

    for (boolean rnn : new boolean[]{true, false}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .weightInit(WeightInit.XAVIER)
                .convolutionMode(ConvolutionMode.Same)
                .list()
                .layer(rnn ? new LSTM.Builder().nOut(3).build() :
                        new Convolution1DLayer.Builder().kernelSize(3).stride(1).nOut(3).build())
                .layer(new BatchNormalization())
                .layer(new RnnOutputLayer.Builder().nOut(3).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build())
                .setInputType(InputType.recurrent(3))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray in = Nd4j.rand(new int[]{1, 3, 5});
        INDArray label = Nd4j.rand(new int[]{1, 3, 5});

        INDArray out = net.output(in);
        assertArrayEquals(new long[]{1, 3, 5}, out.shape());

        net.fit(in, label);
        log.info("OK: {}", (rnn ? "rnn" : "cnn1d"));
    }
}
 

@Test
public void testOutputLayersRnnForwardPass() {
    //Test output layer with RNNs (
    //Expect all outputs etc. to be 2d
    int nIn = 2;
    int nOut = 5;
    int layerSize = 4;
    int timeSeriesLength = 6;
    int miniBatchSize = 3;

    Random r = new Random(12345L);
    INDArray input = Nd4j.zeros(miniBatchSize, nIn, timeSeriesLength);
    for (int i = 0; i < miniBatchSize; i++) {
        for (int j = 0; j < nIn; j++) {
            for (int k = 0; k < timeSeriesLength; k++) {
                input.putScalar(new int[] {i, j, k}, r.nextDouble() - 0.5);
            }
        }
    }

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list()
                    .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize)
                                    .dist(new NormalDistribution(0, 1)).activation(Activation.TANH)
                                    .updater(new NoOp()).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut)
                                    .dist(new NormalDistribution(0, 1))
                                    .updater(new NoOp()).build())
                    .inputPreProcessor(1, new RnnToFeedForwardPreProcessor()).build();

    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();

    INDArray out2d = mln.feedForward(input).get(2);
    assertArrayEquals(out2d.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut});

    INDArray out = mln.output(input);
    assertArrayEquals(out.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut});

    INDArray preout = mln.output(input);
    assertArrayEquals(preout.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut});

    //As above, but for RnnOutputLayer. Expect all activations etc. to be 3d

    MultiLayerConfiguration confRnn = new NeuralNetConfiguration.Builder().seed(12345L).list()
                    .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize)
                                    .dist(new NormalDistribution(0, 1)).activation(Activation.TANH)
                                    .updater(new NoOp()).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut)
                                    .dist(new NormalDistribution(0, 1))
                                    .updater(new NoOp()).build())
                    .build();

    MultiLayerNetwork mlnRnn = new MultiLayerNetwork(confRnn);
    mln.init();

    INDArray out3d = mlnRnn.feedForward(input).get(2);
    assertArrayEquals(out3d.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength});

    INDArray outRnn = mlnRnn.output(input);
    assertArrayEquals(outRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength});

    INDArray preoutRnn = mlnRnn.output(input);
    assertArrayEquals(preoutRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength});
}
 

@Test
public void testDtypesModelVsGlobalDtypeMisc() {
    for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
        Nd4j.setDefaultDataTypes(globalDtype, globalDtype);
        for (DataType networkDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
            assertEquals(globalDtype, Nd4j.dataType());
            assertEquals(globalDtype, Nd4j.defaultFloatingPointType());

            String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype;


            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .dataType(networkDtype)
                    .convolutionMode(ConvolutionMode.Same)
                    .updater(new Adam(1e-2))
                    .list()
                    .layer(new SpaceToBatchLayer.Builder().blocks(1, 1).build())
                    .layer(new SpaceToDepthLayer.Builder().blocks(2).build())
                    .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .setInputType(InputType.convolutional(28, 28, 5))
                    .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            net.initGradientsView();
            assertEquals(msg, networkDtype, net.params().dataType());
            assertEquals(msg, networkDtype, net.getFlattenedGradients().dataType());
            assertEquals(msg, networkDtype, net.getUpdater(true).getStateViewArray().dataType());

            INDArray in = Nd4j.rand(networkDtype, 2, 5, 28, 28);
            INDArray label = TestUtils.randomOneHot(2, 10).castTo(networkDtype);

            INDArray out = net.output(in);
            assertEquals(msg, networkDtype, out.dataType());
            List<INDArray> ff = net.feedForward(in);
            for (int i = 0; i < ff.size(); i++) {
                String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName());
                assertEquals(s, networkDtype, ff.get(i).dataType());
            }

            net.setInput(in);
            net.setLabels(label);
            net.computeGradientAndScore();

            net.fit(new DataSet(in, label));

            logUsedClasses(net);

            //Now, test mismatched dtypes for input/labels:
            for (DataType inputLabelDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
                INDArray in2 = in.castTo(inputLabelDtype);
                INDArray label2 = label.castTo(inputLabelDtype);
                net.output(in2);
                net.setInput(in2);
                net.setLabels(label2);
                net.computeGradientAndScore();

                net.fit(new DataSet(in2, label2));
            }
        }
    }
}
 

@Test
public void testMaskingRnn() {


    int timeSeriesLength = 5;
    int nIn = 5;
    int layerSize = 4;
    int nOut = 2;
    int[] minibatchSizes = new int[] {1, 3};

    for (int miniBatchSize : minibatchSizes) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .updater(new NoOp())
                        .dist(new NormalDistribution(0, 1.0)).seed(12345L).list()
                        .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                                        .build())
                        .layer(1, new org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer.Builder()
                                        .poolingType(PoolingType.AVG).build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).build())
                        .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Random r = new Random(12345L);
        INDArray input = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength}).subi(0.5);

        INDArray mask;
        if (miniBatchSize == 1) {
            mask = Nd4j.create(new double[] {1, 1, 1, 1, 0}).reshape(1,5);
        } else {
            mask = Nd4j.create(new double[][] {{1, 1, 1, 1, 1}, {1, 1, 1, 1, 0}, {1, 1, 1, 0, 0}});
        }

        INDArray labels = Nd4j.zeros(miniBatchSize, nOut);
        for (int i = 0; i < miniBatchSize; i++) {
            int idx = r.nextInt(nOut);
            labels.putScalar(i, idx, 1.0);
        }

        net.setLayerMaskArrays(mask, null);
        INDArray outputMasked = net.output(input);

        net.clearLayerMaskArrays();

        for (int i = 0; i < miniBatchSize; i++) {
            INDArray maskRow = mask.getRow(i);
            int tsLength = maskRow.sumNumber().intValue();
            INDArray inputSubset = input.get(NDArrayIndex.interval(i, i, true), NDArrayIndex.all(),
                            NDArrayIndex.interval(0, tsLength));

            INDArray outSubset = net.output(inputSubset);
            INDArray outputMaskedSubset = outputMasked.getRow(i,true);

            assertEquals(outSubset, outputMaskedSubset);
        }
    }
}
 

@Test
public void testTransferLearningSameDiffLayers(){

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .dataType(DataType.DOUBLE)
            .activation(Activation.TANH)
            .updater(new Adam(0.01))
            .weightInit(WeightInit.XAVIER)
            .list()
            .layer(new LSTM.Builder().nOut(8).build())
            .layer( new SelfAttentionLayer.Builder().nOut(4).nHeads(2).projectInput(true).build())
            .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build())
            .layer(new OutputLayer.Builder().nOut(2).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .setInputType(InputType.recurrent(4))
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray in = Nd4j.rand(DataType.FLOAT, 3, 4, 5);
    INDArray out = net.output(in);

    MultiLayerNetwork net2 = new TransferLearning.Builder(net)
            .fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build())
            .removeLayersFromOutput(1)
            .addLayer(new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();

    net2.setParam("3_W", net.getParam("3_W"));
    net2.setParam("3_b", net.getParam("3_b"));

    Map<String,INDArray> p1 = net.paramTable();
    Map<String,INDArray> p2 = net2.paramTable();
    for(String s : p1.keySet()){
        INDArray i1 = p1.get(s);
        INDArray i2 = p2.get(s);
        assertEquals(s, i1, i2);
    }

    INDArray out2 = net2.output(in);

    assertEquals(out, out2);
}
 

@Test
    public void testSameDiffDenseBackward() {
        int nIn = 3;
        int nOut = 4;

        for (boolean workspaces : new boolean[]{false, true}) {

            for (int minibatch : new int[]{5, 1}) {

                Activation[] afns = new Activation[]{
                        Activation.TANH,
                        Activation.SIGMOID,
                        Activation.ELU,
                        Activation.IDENTITY,
                        Activation.SOFTPLUS,
                        Activation.SOFTSIGN,
                        Activation.HARDTANH,
                        Activation.CUBE,
                        Activation.RELU
                };

                for (Activation a : afns) {
                    log.info("Starting test - " + a + " - minibatch " + minibatch + ", workspaces: " + workspaces);
                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .trainingWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                            .inferenceWorkspaceMode(workspaces ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                            .list()
                            .layer(new SameDiffDense.Builder().nIn(nIn).nOut(nOut)
                                    .activation(a)
                                    .build())
                            .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                            .build();

                    MultiLayerNetwork netSD = new MultiLayerNetwork(conf);
                    netSD.init();

                    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                            .list()
                            .layer(new DenseLayer.Builder().activation(a).nIn(nIn).nOut(nOut).build())
                            .layer(new OutputLayer.Builder().nIn(nOut).nOut(nOut).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                            .build();

                    MultiLayerNetwork netStandard = new MultiLayerNetwork(conf2);
                    netStandard.init();

                    netSD.params().assign(netStandard.params());

                    //Check params:
                    assertEquals(netStandard.params(), netSD.params());
                    assertEquals(netStandard.paramTable(), netSD.paramTable());

                    INDArray in = Nd4j.rand(minibatch, nIn);
                    INDArray l = TestUtils.randomOneHot(minibatch, nOut, 12345);
                    netSD.setInput(in);
                    netStandard.setInput(in);
                    netSD.setLabels(l);
                    netStandard.setLabels(l);

                    netSD.computeGradientAndScore();
                    netStandard.computeGradientAndScore();

                    Gradient gSD = netSD.gradient();
                    Gradient gStd = netStandard.gradient();

                    Map<String, INDArray> m1 = gSD.gradientForVariable();
                    Map<String, INDArray> m2 = gStd.gradientForVariable();

                    assertEquals(m2.keySet(), m1.keySet());

                    for (String s : m1.keySet()) {
                        INDArray i1 = m1.get(s);
                        INDArray i2 = m2.get(s);

                        assertEquals(s, i2, i1);
                    }

                    assertEquals(gStd.gradient(), gSD.gradient());

                    //Sanity check: different minibatch size
                    in = Nd4j.rand(2 * minibatch, nIn);
                    l = TestUtils.randomOneHot(2 * minibatch, nOut, 12345);
                    netSD.setInput(in);
                    netStandard.setInput(in);
                    netSD.setLabels(l);
                    netStandard.setLabels(l);

                    netSD.computeGradientAndScore();
//                    netStandard.computeGradientAndScore();
//                    assertEquals(netStandard.gradient().gradient(), netSD.gradient().gradient());

                    //Sanity check on different minibatch sizes:
                    INDArray newIn = Nd4j.vstack(in, in);
                    INDArray outMbsd = netSD.output(newIn);
                    INDArray outMb = netStandard.output(newIn);
                    assertEquals(outMb, outMbsd);
                }
            }
        }
    }
 

public void runCifar(boolean preProcessCifar) throws Exception {
        final int height = 32;
        final int width = 32;
        int channels = 3;
        int outputNum = CifarLoader.NUM_LABELS;
        int batchSize = 5;
        int seed = 123;
        int listenerFreq = 1;

        Cifar10DataSetIterator cifar = new Cifar10DataSetIterator(batchSize);

        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(channels).nOut(6).weightInit(WeightInit.XAVIER)
                                        .activation(Activation.RELU).build())
                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                        .build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                        .build())

                        .setInputType(InputType.convolutionalFlat(height, width, channels));

        MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
        model.init();

        //model.setListeners(Arrays.asList((TrainingListener) new ScoreIterationListener(listenerFreq)));

        CollectScoresIterationListener listener = new CollectScoresIterationListener(listenerFreq);
        model.setListeners(listener);

        model.fit(cifar);

        cifar = new Cifar10DataSetIterator(batchSize);
        Evaluation eval = new Evaluation(cifar.getLabels());
        while (cifar.hasNext()) {
            DataSet testDS = cifar.next(batchSize);
            INDArray output = model.output(testDS.getFeatures());
            eval.eval(testDS.getLabels(), output);
        }
//        System.out.println(eval.stats(true));
        listener.exportScores(System.out);
    }
 

@Test
public void testAttentionDTypes() {
    for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
        Nd4j.setDefaultDataTypes(globalDtype, globalDtype);
        for (DataType networkDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
            assertEquals(globalDtype, Nd4j.dataType());
            assertEquals(globalDtype, Nd4j.defaultFloatingPointType());

            String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype;

            int mb = 3;
            int nIn = 3;
            int nOut = 5;
            int tsLength = 4;
            int layerSize = 8;
            int numQueries = 6;

            INDArray in = Nd4j.rand(networkDtype, new long[]{mb, nIn, tsLength});
            INDArray labels = TestUtils.randomOneHot(mb, nOut);

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .dataType(networkDtype)
                    .activation(Activation.TANH)
                    .updater(new NoOp())
                    .weightInit(WeightInit.XAVIER)
                    .list()
                    .layer(new LSTM.Builder().nOut(layerSize).build())
                    .layer(new SelfAttentionLayer.Builder().nOut(8).nHeads(2).projectInput(true).build())
                    .layer(new LearnedSelfAttentionLayer.Builder().nOut(8).nHeads(2).nQueries(numQueries).projectInput(true).build())
                    .layer(new RecurrentAttentionLayer.Builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build())
                    .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build())
                    .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX)
                            .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .setInputType(InputType.recurrent(nIn))
                    .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray out = net.output(in);
            assertEquals(msg, networkDtype, out.dataType());
            List<INDArray> ff = net.feedForward(in);
            for (int i = 0; i < ff.size(); i++) {
                String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName());
                assertEquals(s, networkDtype, ff.get(i).dataType());
            }

            net.setInput(in);
            net.setLabels(labels);
            net.computeGradientAndScore();

            net.fit(new DataSet(in, labels));

            logUsedClasses(net);

            //Now, test mismatched dtypes for input/labels:
            for (DataType inputLabelDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
                INDArray in2 = in.castTo(inputLabelDtype);
                INDArray label2 = labels.castTo(inputLabelDtype);
                net.output(in2);
                net.setInput(in2);
                net.setLabels(label2);
                net.computeGradientAndScore();

                net.fit(new DataSet(in2, label2));
            }
        }
    }
}
 

@Test
public void testFeedForwardWithKeyInputMask() {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
            .list()
            .layer( new LSTM.Builder().nIn(4).nOut(3).build())
            .layer(new GlobalPoolingLayer(PoolingType.AVG))
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3)
                    .activation(Activation.SOFTMAX).build())
            .build();


    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    List<org.nd4j.linalg.dataset.DataSet> ds = Arrays.asList(
            new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,0,0})),
            new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,1,0})),
            new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,1,1}))
    );


    Map<Integer,INDArray> expected = new HashMap<>();
    List<Tuple2<Integer, Tuple2<INDArray,INDArray>>> mapFeatures = new ArrayList<>();
    int count = 0;
    int arrayCount = 0;
    Random r = new Random(12345);


    int i=0;
    for(org.nd4j.linalg.dataset.DataSet d : ds){

        INDArray f = d.getFeatures();
        INDArray fm = d.getFeaturesMaskArray();

        mapFeatures.add(new Tuple2<>(i, new Tuple2<>(f, fm)));

        INDArray out = net.output(f, false, fm, null);
        expected.put(i++, out);
    }

    JavaPairRDD<Integer, Tuple2<INDArray,INDArray>> rdd = sc.parallelizePairs(mapFeatures);

    SparkDl4jMultiLayer multiLayer = new SparkDl4jMultiLayer(sc, net, null);
    Map<Integer, INDArray> map = multiLayer.feedForwardWithMaskAndKey(rdd, 16).collectAsMap();

    for (i = 0; i < expected.size(); i++) {
        INDArray exp = expected.get(i);
        INDArray act = map.get(i);

        assertEquals(exp, act);
    }
}