Java Code Examples for org.deeplearning4j.nn.multilayer.MultiLayerNetwork#setParams()

@Test
public void testScoringDataSet() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();

    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same score
    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());

    double scoreMLN = net.score(ds, false);
    double scoreCG = graph.score(ds, false);

    assertEquals(scoreMLN, scoreCG, 1e-4);
}
 

protected Model buildMultiLayerNetworkModel(int numFeatures) throws Exception {

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder().nIn(numFeatures).nOut(1).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[numFeatures+1];
    float base = 1f;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 

@Override
public MultiLayerNetwork getInitialModel() {
    if(workerTogglePeriodicGC != null)
        Nd4j.getMemoryManager().togglePeriodicGc(workerTogglePeriodicGC);
    if(workerPeriodicGCFrequency != null)
        Nd4j.getMemoryManager().setAutoGcWindow(workerPeriodicGCFrequency);

    // This method will be called ONLY once, in master thread
    //Before getting NetBroadcastTuple, to ensure it always gets mapped to device 0
    Nd4j.getAffinityManager().unsafeSetDevice(0);

    NetBroadcastTuple tuple = broadcastModel.getValue();
    if (tuple.getConfiguration() != null) {
        MultiLayerConfiguration conf = tuple.getConfiguration();
        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();

        if (tuple.getParameters() != null)
            network.setParams(tuple.getParameters());

        // we can assign properly, without
        if (tuple.getUpdaterState() != null)
            network.getUpdater().getStateViewArray().assign(tuple.getUpdaterState());

        return network;
    } else
        return null;
}
 

@Test
public void testIrisFit() {

    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlnConfig = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlnConfig);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);

    graph.setParams(params.dup());
    net.setParams(params.dup());


    DataSetIterator iris = new IrisDataSetIterator(75, 150);

    net.fit(iris);
    iris.reset();

    graph.fit(iris);

    //Check that parameters are equal for both models after fitting:
    INDArray paramsMLN = net.params();
    INDArray paramsGraph = graph.params();

    assertNotEquals(params, paramsGraph);
    assertEquals(paramsMLN, paramsGraph);
}
 

protected Model buildMultiLayerNetworkModel(int numInputs, int numOutputs) throws Exception {

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[(numInputs+1)*numOutputs];
    final float base0 = 0.01f;
    float base = base0;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      if (base > 1/base0) base = base0;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 

@Test
public void testForwardBasicIris() {

    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();

    graph.setInput(0, ds.getFeatures());
    Map<String, INDArray> activations = graph.feedForward(false);
    assertEquals(3, activations.size()); //2 layers + 1 input node
    assertTrue(activations.containsKey("input"));
    assertTrue(activations.containsKey("firstLayer"));
    assertTrue(activations.containsKey("outputLayer"));

    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs
    Nd4j.getRandom().setSeed(12345);
    int nParams = getNumParams();
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());

    List<INDArray> mlnAct = net.feedForward(ds.getFeatures(), false);
    activations = graph.feedForward(ds.getFeatures(), false);

    assertEquals(mlnAct.get(0), activations.get("input"));
    assertEquals(mlnAct.get(1), activations.get("firstLayer"));
    assertEquals(mlnAct.get(2), activations.get("outputLayer"));
}
 

private static Model buildModel() throws Exception {

    final int numInputs = 3;
    final int numOutputs = 2;

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 };
    // positive weight for first output, negative weight for second output, no biases
    assertEquals((numInputs+1)*numOutputs, floats.length);

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 

@Test
public void testEmbeddingBackwardPass() {
    //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation
    // input with a DenseLayer

    int nClassesIn = 10;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
            .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1,
                    new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4)
                            .activation(Activation.SOFTMAX).build())
            .build();
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
            .weightInit(WeightInit.XAVIER).list()
            .layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4)
                            .activation(Activation.SOFTMAX).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net.init();
    net2.init();

    net2.setParams(net.params().dup());

    int batchSize = 3;
    INDArray inEmbedding = Nd4j.create(batchSize, 1);
    INDArray inOneHot = Nd4j.create(batchSize, nClassesIn);
    INDArray outLabels = Nd4j.create(batchSize, 4);

    Random r = new Random(12345);
    for (int i = 0; i < batchSize; i++) {
        int classIdx = r.nextInt(nClassesIn);
        inEmbedding.putScalar(i, classIdx);
        inOneHot.putScalar(new int[]{i, classIdx}, 1.0);

        int labelIdx = r.nextInt(4);
        outLabels.putScalar(new int[]{i, labelIdx}, 1.0);
    }

    net.setInput(inEmbedding);
    net2.setInput(inOneHot);
    net.setLabels(outLabels);
    net2.setLabels(outLabels);

    net.computeGradientAndScore();
    net2.computeGradientAndScore();

    assertEquals(net2.score(), net.score(), 1e-6);

    Map<String, INDArray> gradient = net.gradient().gradientForVariable();
    Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
    assertEquals(gradient.size(), gradient2.size());

    for (String s : gradient.keySet()) {
        assertEquals(gradient2.get(s), gradient.get(s));
    }
}
 

@Test
public void testBackwardIrisBasic() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();

    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs
    Nd4j.getRandom().setSeed(12345);
    int nParams = (4 * 5 + 5) + (5 * 3 + 3);
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());

    INDArray input = ds.getFeatures();
    INDArray labels = ds.getLabels();
    graph.setInput(0, input.dup());
    graph.setLabel(0, labels.dup());

    net.setInput(input.dup());
    net.setLabels(labels.dup());

    //Compute gradients
    net.computeGradientAndScore();
    Pair<Gradient, Double> netGradScore = net.gradientAndScore();

    graph.computeGradientAndScore();
    Pair<Gradient, Double> graphGradScore = graph.gradientAndScore();

    assertEquals(netGradScore.getSecond(), graphGradScore.getSecond(), 1e-3);

    //Compare gradients
    Gradient netGrad = netGradScore.getFirst();
    Gradient graphGrad = graphGradScore.getFirst();

    assertNotNull(graphGrad);
    assertEquals(netGrad.gradientForVariable().size(), graphGrad.gradientForVariable().size());

    assertEquals(netGrad.getGradientFor("0_W"), graphGrad.getGradientFor("firstLayer_W"));
    assertEquals(netGrad.getGradientFor("0_b"), graphGrad.getGradientFor("firstLayer_b"));
    assertEquals(netGrad.getGradientFor("1_W"), graphGrad.getGradientFor("outputLayer_W"));
    assertEquals(netGrad.getGradientFor("1_b"), graphGrad.getGradientFor("outputLayer_b"));
}
 

@Test
public void testEmbeddingSequenceLayerWithMasking() {
    //Idea: have masking on the input with an embedding and dense layers on input
    //Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked

    int[] miniBatchSizes = {1, 3};
    int nIn = 2;
    Random r = new Random(12345);

    int numInputClasses = 10;
    int timeSeriesLength = 5;

    for (DataType maskDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) {
        for (DataType inLabelDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) {
            for(int inputRank : new int[]{2, 3}) {
                for (int nExamples : miniBatchSizes) {
                    Nd4j.getRandom().setSeed(12345);

                    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .updater(new Sgd(0.1)).seed(12345).list()
                            .layer(0, new EmbeddingSequenceLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses)
                                    .nOut(5).build())
                            .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                            .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                            .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                    .nOut(4).build())
                            .setInputType(InputType.recurrent(1)).build();

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .updater(new Sgd(0.1)).seed(12345).list()
                            .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5)
                                    .build())
                            .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                            .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                            .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                    .nOut(4).build())
                            .setInputType(InputType.recurrent(1)).build();

                    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                    net2.init();

                    net2.setParams(net.params().dup());

                    INDArray inEmbedding = Nd4j.zeros(inLabelDtype, inputRank == 2 ? new long[]{nExamples, timeSeriesLength} : new long[]{nExamples, 1, timeSeriesLength});
                    INDArray inDense = Nd4j.zeros(inLabelDtype, nExamples, numInputClasses, timeSeriesLength);

                    INDArray labels = Nd4j.zeros(inLabelDtype, nExamples, 4, timeSeriesLength);

                    for (int i = 0; i < nExamples; i++) {
                        for (int j = 0; j < timeSeriesLength; j++) {
                            int inIdx = r.nextInt(numInputClasses);
                            inEmbedding.putScalar(inputRank == 2 ? new int[]{i, j} : new int[]{i, 0, j}, inIdx);
                            inDense.putScalar(new int[]{i, inIdx, j}, 1.0);

                            int outIdx = r.nextInt(4);
                            labels.putScalar(new int[]{i, outIdx, j}, 1.0);
                        }
                    }

                    INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength);
                    for (int i = 0; i < nExamples; i++) {
                        for (int j = 0; j < timeSeriesLength; j++) {
                            inputMask.putScalar(new int[]{i, j}, (r.nextBoolean() ? 1.0 : 0.0));
                        }
                    }

                    net.setLayerMaskArrays(inputMask, null);
                    net2.setLayerMaskArrays(inputMask, null);
                    List<INDArray> actEmbedding = net.feedForward(inEmbedding, false);
                    List<INDArray> actDense = net2.feedForward(inDense, false);
                    for (int i = 2; i < actEmbedding.size(); i++) { //Start from layer 2: EmbeddingSequence is 3d, first dense is 2d (before reshape)
                        assertEquals(actDense.get(i), actEmbedding.get(i));
                    }

                    net.setLabels(labels);
                    net2.setLabels(labels);
                    net.computeGradientAndScore();
                    net2.computeGradientAndScore();

                    assertEquals(net2.score(), net.score(), 1e-5);

                    Map<String, INDArray> gradients = net.gradient().gradientForVariable();
                    Map<String, INDArray> gradients2 = net2.gradient().gradientForVariable();
                    assertEquals(gradients.keySet(), gradients2.keySet());
                    for (String s : gradients.keySet()) {
                        assertEquals(gradients2.get(s), gradients.get(s));
                    }
                }
            }
        }
    }
}
 

@Test
    public void testEmbeddingLayerWithMasking() {
        //Idea: have masking on the input with an embedding and dense layers on input
        //Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked

        int[] miniBatchSizes = {1, 2, 5};
        int nIn = 2;
        Random r = new Random(12345);

        int numInputClasses = 10;
        int timeSeriesLength = 5;

        for (DataType maskDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) {
            for (int nExamples : miniBatchSizes) {
                Nd4j.getRandom().setSeed(12345);

                MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .updater(new Sgd(0.1)).seed(12345).list()
                        .layer(0, new EmbeddingLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses)
                                .nOut(5).build())
                        .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                        .layer(2, new GravesLSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                        .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                .nOut(4).build())
                        .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                        .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .updater(new Sgd(0.1)).seed(12345).list()
                        .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5)
                                .build())
                        .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build())
                        .layer(2, new GravesLSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build())
                        .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                                .nOut(4).build())
                        .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                        .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).build();

                MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
                net2.init();

                net2.setParams(net.params().dup());

                INDArray inEmbedding = Nd4j.zeros(nExamples, 1, timeSeriesLength);
                INDArray inDense = Nd4j.zeros(nExamples, numInputClasses, timeSeriesLength);

                INDArray labels = Nd4j.zeros(nExamples, 4, timeSeriesLength);

                for (int i = 0; i < nExamples; i++) {
                    for (int j = 0; j < timeSeriesLength; j++) {
                        int inIdx = r.nextInt(numInputClasses);
                        inEmbedding.putScalar(new int[]{i, 0, j}, inIdx);
                        inDense.putScalar(new int[]{i, inIdx, j}, 1.0);

                        int outIdx = r.nextInt(4);
                        labels.putScalar(new int[]{i, outIdx, j}, 1.0);
                    }
                }

                INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength);
                for (int i = 0; i < nExamples; i++) {
                    for (int j = 0; j < timeSeriesLength; j++) {
                        inputMask.putScalar(new int[]{i, j}, (r.nextBoolean() ? 1.0 : 0.0));
                    }
                }

                net.setLayerMaskArrays(inputMask, null);
                net2.setLayerMaskArrays(inputMask, null);
                List<INDArray> actEmbedding = net.feedForward(inEmbedding, false);
                List<INDArray> actDense = net2.feedForward(inDense, false);
                for (int i = 1; i < actEmbedding.size(); i++) {
                    assertEquals(actDense.get(i), actEmbedding.get(i));
                }

                net.setLabels(labels);
                net2.setLabels(labels);
                net.computeGradientAndScore();
                net2.computeGradientAndScore();

//                System.out.println(net.score() + "\t" + net2.score());
                assertEquals(net2.score(), net.score(), 1e-5);

                Map<String, INDArray> gradients = net.gradient().gradientForVariable();
                Map<String, INDArray> gradients2 = net2.gradient().gradientForVariable();
                assertEquals(gradients.keySet(), gradients2.keySet());
                for (String s : gradients.keySet()) {
                    assertEquals(gradients2.get(s), gradients.get(s));
                }
            }
        }
    }
 

@Test
    public void testEmbeddingLayerRNN() {

        int nClassesIn = 10;

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
                .dataType(DataType.DOUBLE)
                .list()
                .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build())
                .layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build())
                .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4)
                        .activation(Activation.SOFTMAX).build())
                .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                .inputPreProcessor(1, new FeedForwardToRnnPreProcessor())
                .build();
        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH)
                .weightInit(WeightInit.XAVIER)
                .dataType(DataType.DOUBLE)
                .list()
                .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build())
                .layer(1, new GravesLSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build())
                .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4)
                        .activation(Activation.SOFTMAX).build())
                .inputPreProcessor(0, new RnnToFeedForwardPreProcessor())
                .inputPreProcessor(1, new FeedForwardToRnnPreProcessor())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net.init();
        net2.init();

        net2.setParams(net.params().dup());

        int batchSize = 3;
        int timeSeriesLength = 8;
        INDArray inEmbedding = Nd4j.create(batchSize, 1, timeSeriesLength);
        INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, timeSeriesLength);
        INDArray outLabels = Nd4j.create(batchSize, 4, timeSeriesLength);

        Random r = new Random(12345);
        for (int i = 0; i < batchSize; i++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                int classIdx = r.nextInt(nClassesIn);
                inEmbedding.putScalar(new int[]{i, 0, j}, classIdx);
                inOneHot.putScalar(new int[]{i, classIdx, j}, 1.0);

                int labelIdx = r.nextInt(4);
                outLabels.putScalar(new int[]{i, labelIdx, j}, 1.0);
            }
        }

        net.setInput(inEmbedding);
        net2.setInput(inOneHot);
        net.setLabels(outLabels);
        net2.setLabels(outLabels);

        net.computeGradientAndScore();
        net2.computeGradientAndScore();

//        System.out.println(net.score() + "\t" + net2.score());
        assertEquals(net2.score(), net.score(), 1e-5);

        Map<String, INDArray> gradient = net.gradient().gradientForVariable();
        Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
        assertEquals(gradient.size(), gradient2.size());

        for (String s : gradient.keySet()) {
            assertEquals(gradient2.get(s), gradient.get(s));
        }

    }
 

@Test
    public void testEmbeddingSequenceBackwardPass() {
        int nClassesIn = 10;
        int embeddingDim = 5;
        int nOut = 4;
        int inputLength = 1;

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength)
                        .hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build())
                .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build())
                .build();
        MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(embeddingDim).activation(Activation.IDENTITY).build())
                .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build())
                .setInputType(InputType.recurrent(nClassesIn))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
        net.init();
        net2.init();

        net2.setParams(net.params().dup());

        int batchSize = 3;
        INDArray inEmbedding = Nd4j.create(batchSize, 1);
        INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, 1);
        INDArray outLabels = Nd4j.create(batchSize, 4, 1);

        Random r = new Random(1337);
        for (int i = 0; i < batchSize; i++) {
            int classIdx = r.nextInt(nClassesIn);
            inEmbedding.putScalar(i, classIdx);
            inOneHot.putScalar(new int[]{i, classIdx, 0}, 1.0);

            int labelIdx = r.nextInt(4);
            outLabels.putScalar(new int[]{i, labelIdx, 0}, 1.0);
        }

        net.setInput(inEmbedding);
        net2.setInput(inOneHot);
        net.setLabels(outLabels);
        net2.setLabels(outLabels);

        net.computeGradientAndScore();
        net2.computeGradientAndScore();

//        System.out.println(net.score() + "\t" + net2.score());
        assertEquals(net2.score(), net.score(), 1e-6);

        Map<String, INDArray> gradient = net.gradient().gradientForVariable();
        Map<String, INDArray> gradient2 = net2.gradient().gradientForVariable();
        assertEquals(gradient.size(), gradient2.size());

        for (String s : gradient.keySet()) {
            assertEquals(gradient2.get(s), gradient.get(s));
        }
    }
 

@Test
    public void testEvalSplitting(){
        //Test for "tbptt-like" functionality

        for(WorkspaceMode ws : WorkspaceMode.values()) {
            System.out.println("Starting test for workspace mode: " + ws);

            int nIn = 4;
            int layerSize = 5;
            int nOut = 6;
            int tbpttLength = 10;
            int tsLength = 5 * tbpttLength + tbpttLength / 2;

            MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                    .seed(12345)
                    .trainingWorkspaceMode(ws)
                    .inferenceWorkspaceMode(ws)
                    .list()
                    .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build())
                    .layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut)
                            .activation(Activation.SOFTMAX)
                            .build())
                    .build();

            MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
                    .seed(12345)
                    .trainingWorkspaceMode(ws)
                    .inferenceWorkspaceMode(ws)
                    .list()
                    .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build())
                    .layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut)
                            .activation(Activation.SOFTMAX).build())
                    .tBPTTLength(10)
                    .backpropType(BackpropType.TruncatedBPTT)
                    .build();

            MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
            net1.init();

            MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
            net2.init();

            net2.setParams(net1.params());

            for(boolean useMask : new boolean[]{false, true}) {

                INDArray in1 = Nd4j.rand(new int[]{3, nIn, tsLength});
                INDArray out1 = TestUtils.randomOneHotTimeSeries(3, nOut, tsLength);

                INDArray in2 = Nd4j.rand(new int[]{5, nIn, tsLength});
                INDArray out2 = TestUtils.randomOneHotTimeSeries(5, nOut, tsLength);

                INDArray lMask1 = null;
                INDArray lMask2 = null;
                if(useMask){
                    lMask1 = Nd4j.create(3, tsLength);
                    lMask2 = Nd4j.create(5, tsLength);
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask1, 0.5));
                    Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask2, 0.5));
                }

                List<DataSet> l = Arrays.asList(new DataSet(in1, out1, null, lMask1), new DataSet(in2, out2, null, lMask2));
                DataSetIterator iter = new ExistingDataSetIterator(l);

//                System.out.println("Net 1 eval");
                org.nd4j.evaluation.IEvaluation[] e1 = net1.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation());
//                System.out.println("Net 2 eval");
                org.nd4j.evaluation.IEvaluation[] e2 = net2.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation());

                assertEquals(e1[0], e2[0]);
                assertEquals(e1[1], e2[1]);
                assertEquals(e1[2], e2[2]);
            }
        }
    }
 

@Test
public void testCnnLossLayer(){

    for(WorkspaceMode ws : WorkspaceMode.values()) {
        log.info("*** Testing workspace: " + ws);

        for (Activation a : new Activation[]{Activation.TANH, Activation.SELU}) {
            //Check that (A+identity) is equal to (identity+A), for activation A
            //i.e., should get same output and weight gradients for both

            MultiLayerConfiguration conf1 =
                    new NeuralNetConfiguration.Builder().seed(12345L)
                            .updater(new NoOp())
                            .convolutionMode(ConvolutionMode.Same)
                            .inferenceWorkspaceMode(ws)
                            .trainingWorkspaceMode(ws)
                            .list()
                            .layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY)
                                    .kernelSize(2, 2).stride(1, 1)
                                    .dist(new NormalDistribution(0, 1.0))
                                    .updater(new NoOp()).build())
                            .layer(new CnnLossLayer.Builder(LossFunction.MSE)
                                    .activation(a)
                                    .build())
                            .build();

            MultiLayerConfiguration conf2 =
                    new NeuralNetConfiguration.Builder().seed(12345L)
                            .updater(new NoOp())
                            .convolutionMode(ConvolutionMode.Same)
                            .inferenceWorkspaceMode(ws)
                            .trainingWorkspaceMode(ws)
                            .list()
                            .layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a)
                                    .kernelSize(2, 2).stride(1, 1)
                                    .dist(new NormalDistribution(0, 1.0))
                                    .updater(new NoOp()).build())
                            .layer(new CnnLossLayer.Builder(LossFunction.MSE)
                                    .activation(Activation.IDENTITY)
                                    .build())
                            .build();

            MultiLayerNetwork mln = new MultiLayerNetwork(conf1);
            mln.init();

            MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2);
            mln2.init();


            mln2.setParams(mln.params());


            INDArray in = Nd4j.rand(new int[]{3, 3, 5, 5});

            INDArray out1 = mln.output(in);
            INDArray out2 = mln2.output(in);

            assertEquals(out1, out2);

            INDArray labels = Nd4j.rand(out1.shape());

            mln.setInput(in);
            mln.setLabels(labels);

            mln2.setInput(in);
            mln2.setLabels(labels);

            mln.computeGradientAndScore();
            mln2.computeGradientAndScore();

            assertEquals(mln.score(), mln2.score(), 1e-6);
            assertEquals(mln.gradient().gradient(), mln2.gradient().gradient());

            //Also check computeScoreForExamples
            INDArray in2a = Nd4j.rand(new int[]{1, 3, 5, 5});
            INDArray labels2a = Nd4j.rand(new int[]{1, 4, 5, 5});

            INDArray in2 = Nd4j.concat(0, in2a, in2a);
            INDArray labels2 = Nd4j.concat(0, labels2a, labels2a);

            INDArray s = mln.scoreExamples(new DataSet(in2, labels2), false);
            assertArrayEquals(new long[]{2, 1}, s.shape());
            assertEquals(s.getDouble(0), s.getDouble(1), 1e-6);

            TestUtils.testModelSerialization(mln);
        }
    }
}
 

@Test
public void testCompareRnnOutputRnnLoss(){
    Nd4j.getRandom().setSeed(12345);

    int timeSeriesLength = 4;
    int nIn = 5;
    int layerSize = 6;
    int nOut = 6;
    int miniBatchSize = 3;

    MultiLayerConfiguration conf1 =
            new NeuralNetConfiguration.Builder().seed(12345L)
                    .updater(new NoOp())
                    .list()
                    .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                            .dist(new NormalDistribution(0, 1.0))
                            .updater(new NoOp()).build())
                    .layer(new DenseLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.IDENTITY).build())
                    .layer(new RnnLossLayer.Builder(LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX)
                            .build())
                    .build();

    MultiLayerNetwork mln = new MultiLayerNetwork(conf1);
    mln.init();


    MultiLayerConfiguration conf2 =
            new NeuralNetConfiguration.Builder().seed(12345L)
                    .updater(new NoOp())
                    .list()
                    .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH)
                            .dist(new NormalDistribution(0, 1.0))
                            .updater(new NoOp()).build())
                    .layer(new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX)
                            .nIn(layerSize).nOut(nOut)
                            .build())
                    .build();

    MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2);
    mln2.init();

    mln2.setParams(mln.params());

    INDArray in = Nd4j.rand(new int[]{miniBatchSize, nIn, timeSeriesLength});

    INDArray out1 = mln.output(in);
    INDArray out2 = mln.output(in);

    assertEquals(out1, out2);

    Random r = new Random(12345);
    INDArray labels = Nd4j.create(miniBatchSize, nOut, timeSeriesLength);
    for( int i=0; i<miniBatchSize; i++ ){
        for( int j=0; j<timeSeriesLength; j++ ){
            labels.putScalar(i, r.nextInt(nOut), j, 1.0);
        }
    }

    mln.setInput(in);
    mln.setLabels(labels);

    mln2.setInput(in);
    mln2.setLabels(labels);

    mln.computeGradientAndScore();
    mln2.computeGradientAndScore();

    assertEquals(mln.gradient().gradient(), mln2.gradient().gradient());
    assertEquals(mln.score(), mln2.score(), 1e-6);

    TestUtils.testModelSerialization(mln);
}
 

@Test
public void testRemoveAndAdd() {
    Nd4j.getRandom().setSeed(12345);
    DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT,10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3));

    NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1));
    FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build();

    MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(//overallConf.list()
                    equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build())
                                    .layer(1, new DenseLayer.Builder().nIn(5).nOut(2).build())
                                    .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                                    LossFunctions.LossFunction.MCXENT)
                                                                    .activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                                    .build())
                                    .build());
    modelToFineTune.init();

    MultiLayerNetwork modelNow =
                    new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf)
                                    .nOutReplace(0, 7, WeightInit.XAVIER, WeightInit.XAVIER)
                                    .nOutReplace(2, 5, WeightInit.XAVIER).removeOutputLayer()
                                    .addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5)
                                                    .nOut(3).updater(new Sgd(0.5)).activation(Activation.SOFTMAX)
                                                    .build())
                                    .build();

    MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(7).build())
                    .layer(1, new DenseLayer.Builder().nIn(7).nOut(2).build())
                    .layer(2, new DenseLayer.Builder().nIn(2).nOut(5).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX)
                                                    .updater(new Sgd(0.5)).nIn(5).nOut(3).build())
                    .build());
    modelExpectedArch.init();

    //modelNow should have the same architecture as modelExpectedArch
    assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape());

    modelNow.setParams(modelExpectedArch.params());
    //fit should give the same results
    modelExpectedArch.fit(randomData);
    modelNow.fit(randomData);
    double scoreExpected = modelExpectedArch.score();
    double scoreActual = modelNow.score();
    assertEquals(scoreExpected, scoreActual, 1e-4);
    assertEquals(modelExpectedArch.params(), modelNow.params());
}
 

@Test
public void testNoutChanges() {
    Nd4j.getRandom().setSeed(12345);
    DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT,10, 2));

    NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1));
    FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1))
                    .build();

    MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(equivalentConf.list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build())
                    .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build())
                    .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build());
    modelToFineTune.init();
    MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf)
                    .nOutReplace(3, 2, WeightInit.XAVIER, WeightInit.XAVIER)
                    .nOutReplace(0, 3, WeightInit.XAVIER, new NormalDistribution(1, 1e-1)).build();

    MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build())
                    .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build())
                    .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2)
                                                    .build())
                    .build());
    modelExpectedArch.init();

    //Will fail - expected because of dist and weight init changes
    //assertEquals(modelExpectedArch.getLayerWiseConfigurations().toJson(), modelNow.getLayerWiseConfigurations().toJson());

    BaseLayer bl0 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(0).getLayer());
    BaseLayer bl1 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(1).getLayer());
    BaseLayer bl3 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(3).getLayer());
    assertEquals(bl0.getWeightInitFn().getClass(), WeightInitXavier.class);
    try {
        assertEquals(JsonMappers.getMapper().writeValueAsString(bl1.getWeightInitFn()),
                JsonMappers.getMapper().writeValueAsString(new WeightInitDistribution(new NormalDistribution(1, 1e-1))));
    } catch (JsonProcessingException e) {
        throw new RuntimeException(e);
    }
    assertEquals(bl3.getWeightInitFn(), new WeightInitXavier());

    //modelNow should have the same architecture as modelExpectedArch
    assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape());
    assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape());

    modelNow.setParams(modelExpectedArch.params());
    //fit should give the same results
    modelExpectedArch.fit(randomData);
    modelNow.fit(randomData);
    assertEquals(modelExpectedArch.score(), modelNow.score(), 0.000001);
    assertEquals(modelExpectedArch.params(), modelNow.params());
}
 

@Test
public void simpleFineTune() {

    long rng = 12345L;
    Nd4j.getRandom().setSeed(rng);
    DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3));
    //original conf
    NeuralNetConfiguration.Builder confToChange =
                    new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.LBFGS)
                                    .updater(new Nesterovs(0.01, 0.99));

    MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(confToChange.list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build());
    modelToFineTune.init();

    //model after applying changes with transfer learning
    MultiLayerNetwork modelNow =
            new TransferLearning.Builder(modelToFineTune)
                    .fineTuneConfiguration(new FineTuneConfiguration.Builder().seed(rng)
                            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                            .updater(new RmsProp(0.5)) //Intent: override both weight and bias LR, unless bias LR is manually set also
                            .l2(0.4).build())
                    .build();

    for (org.deeplearning4j.nn.api.Layer l : modelNow.getLayers()) {
        BaseLayer bl = ((BaseLayer) l.conf().getLayer());
        assertEquals(new RmsProp(0.5), bl.getIUpdater());
    }


    NeuralNetConfiguration.Builder confSet = new NeuralNetConfiguration.Builder().seed(rng)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new RmsProp(0.5)).l2(0.4);

    MultiLayerNetwork expectedModel = new MultiLayerNetwork(confSet.list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build());
    expectedModel.init();
    expectedModel.setParams(modelToFineTune.params().dup());

    assertEquals(expectedModel.params(), modelNow.params());

    //Check json
    MultiLayerConfiguration expectedConf = expectedModel.getLayerWiseConfigurations();
    assertEquals(expectedConf.toJson(), modelNow.getLayerWiseConfigurations().toJson());

    //Check params after fit
    modelNow.fit(randomData);
    expectedModel.fit(randomData);

    assertEquals(modelNow.score(), expectedModel.score(), 1e-6);
    INDArray pExp = expectedModel.params();
    INDArray pNow = modelNow.params();
    assertEquals(pExp, pNow);
}
 

@Test
public void testGetSetParams() {

    MultiLayerNetwork net = getCNNMLNConfig(true, false);

    INDArray paramsOrig = net.params().dup();
    net.setParams(paramsOrig);

    INDArray params2 = net.params();

    assertEquals(paramsOrig, params2);
}