org.deeplearning4j.nn.weights.WeightInit Java Examples

private static ComputationGraphConfiguration getGraphConfCNN(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "in")
                    .addLayer("1", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "0")
                    .addLayer("2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10)
                                    .build(), "1")
                    .setOutputs("2").setInputTypes(InputType.convolutional(10, 10, 3))
                    .build();
    return conf;
}
 

@Test
public void testMultiCNNLayer() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new LocalResponseNormalization.Builder().build()).layer(2,
                                    new DenseLayer.Builder()
                                                    .nOut(2).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10)
                                    .build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    network.fit(next);
}
 

@Test
public void testPredict() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .weightInit(WeightInit.XAVIER).seed(12345L).list()
                    .layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build())
                    .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(50).nOut(10).build())
                    .setInputType(InputType.convolutional(28, 28, 1)).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator ds = new MnistDataSetIterator(10, 10);
    net.fit(ds);

    DataSetIterator testDs = new MnistDataSetIterator(1, 1);
    DataSet testData = testDs.next();
    testData.setLabelNames(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"));
    String actualLables = testData.getLabelName(0);
    List<String> prediction = net.predict(testData);
    assertTrue(actualLables != null);
    assertTrue(prediction.get(0) != null);
}
 

public static void main(String[] args) throws Exception {
    //Define a simple MultiLayerNetwork:
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9))
        .list()
        .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.TANH).build())
        .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(3).nOut(3).build())
        .backprop(true).pretrain(false).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();


    //Save the model
    File locationToSave = new File("model/MyMultiLayerNetwork.zip");      //Where to save the network. Note: the file is in .zip format - can be opened externally
    /**
     * 主要是用于保存模型的更新器信息
     * 如果模型保存之后还打算继续训练，则进行保存 -> true 才能根据后面的数据进行增量更新
     * 如果不打算继续训练 -> 模型定型之后，false
     */
    boolean saveUpdater = true;                                             //Updater: i.e., the state for Momentum, RMSProp, Adagrad etc. Save this if you want to train your network more in the future
    ModelSerializer.writeModel(net, locationToSave, saveUpdater);

    //Load the model
    MultiLayerNetwork restored = ModelSerializer.restoreMultiLayerNetwork(locationToSave);


    System.out.println("Saved and loaded parameters are equal:      " + net.params().equals(restored.params()));
    System.out.println("Saved and loaded configurations are equal:  " + net.getLayerWiseConfigurations().equals(restored.getLayerWiseConfigurations()));
}
 

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(10.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY)
                                    .lossFunction(LossFunctions.LossFunction.MSE).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(5));

    JavaRDD<DataSet> irisData = getIris();
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5
                                    .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc()))
                                    .modelSaver(saver).build();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new SparkEarlyStoppingTrainer(getContext().sc(),
                    new ParameterAveragingTrainingMaster(true, 4, 1, 150 / 4, 1, 0), esConf, net, irisData);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(7.5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 

@Test
    public void testLfwModel() throws Exception {
        final int numRows = 28;
        final int numColumns = 28;
        int numChannels = 3;
        int outputNum = LFWLoader.NUM_LABELS;
        int numSamples = LFWLoader.NUM_IMAGES;
        int batchSize = 2;
        int seed = 123;
        int listenerFreq = 1;

        LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples,
                        new int[] {numRows, numColumns, numChannels}, outputNum, false, true, 1.0, new Random(seed));

        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6)
                                        .weightInit(WeightInit.XAVIER).activation(Activation.RELU).build())
                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                        .stride(1, 1).build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                        .build())
                        .setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels))
                        ;

        MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
        model.init();

        model.setListeners(new ScoreIterationListener(listenerFreq));

        model.fit(lfw.next());

        DataSet dataTest = lfw.next();
        INDArray output = model.output(dataTest.getFeatures());
        Evaluation eval = new Evaluation(outputNum);
        eval.eval(dataTest.getLabels(), output);
//        System.out.println(eval.stats());
    }
 

public static ComputationGraphConfiguration.GraphBuilder addLayers(ComputationGraphConfiguration.GraphBuilder graphBuilder, int layerNumber, String input, int filterSize, int nIn, int nOut, int poolSize, int poolStride) {
    graphBuilder
            .addLayer("convolution2d_" + layerNumber,
                    new ConvolutionLayer.Builder(filterSize,filterSize)
                            .nIn(nIn)
                            .nOut(nOut)
                            .weightInit(WeightInit.XAVIER)
                            .convolutionMode(ConvolutionMode.Same)
                            .hasBias(false)
                            .stride(1,1)
                            .activation(Activation.IDENTITY)
                            .build(),
                    input)
            .addLayer("batchnormalization_" + layerNumber,
                    new BatchNormalization.Builder()
                            .nIn(nOut).nOut(nOut)
                            .weightInit(WeightInit.XAVIER)
                            .activation(Activation.IDENTITY)
                            .build(),
                    "convolution2d_" + layerNumber)
            .addLayer("activation_" + layerNumber,
                    new ActivationLayer.Builder()
                            .activation(new ActivationLReLU(0.1))
                            .build(),
                    "batchnormalization_" + layerNumber);
    if (poolSize > 0) {
        graphBuilder
                .addLayer("maxpooling2d_" + layerNumber,
                        new SubsamplingLayer.Builder()
                                .kernelSize(poolSize, poolSize)
                                .stride(poolStride, poolStride)
                                .convolutionMode(ConvolutionMode.Same)
                                .build(),
                        "activation_" + layerNumber);
    }

    return graphBuilder;
}
 

public GraphBuilder nOutReplace(String layerName, int nOut, WeightInit scheme, Distribution dist) {
    if(scheme == WeightInit.DISTRIBUTION) {
        throw new UnsupportedOperationException("Not supported!, Use " +
                "nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext)) instead!");
    }
    return nOutReplace(layerName, nOut, scheme.getWeightInitFunction(), new WeightInitDistribution(dist));
}
 

@Test
public void testChangeNOutNIn() {
    final String inputName = "input";
    final String changeNoutName = "changeNout";
    final String poolName = "pool";
    final String afterPoolName = "afterPool";
    final String outputName = "output";
    final INDArray input = Nd4j.create(new long[] {1, 2, 4, 4});
    final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .addInputs(inputName)
            .setOutputs(outputName)
            .setInputTypes(InputType.inferInputTypes(input))
            .addLayer(changeNoutName, new Convolution2D.Builder(1, 1)
                    .nOut(10)
                    .build(), inputName)
            .addLayer(poolName, new SubsamplingLayer.Builder(1,1).build(), changeNoutName)
            .addLayer(afterPoolName, new Convolution2D.Builder(1, 1)
                    .nOut(7)
                    .build(), poolName)
            .addLayer(outputName, new OutputLayer.Builder()
                    .activation(Activation.SOFTMAX)
                    .nOut(2)
                    .build(), afterPoolName)
            .build());
    graph.init();

    final ComputationGraph newGraph = new TransferLearning.GraphBuilder(graph)
            .nOutReplace(changeNoutName, 5, WeightInit.XAVIER)
            .nInReplace(afterPoolName, 5, WeightInit.XAVIER)
            .build();

    newGraph.init();

    assertEquals("Incorrect number of outputs!", 5 , newGraph.layerSize(changeNoutName));
    assertEquals("Incorrect number of inputs!", 5, newGraph.layerInputSize(afterPoolName));
    newGraph.output(input);
}
 

public GraphBuilder nOutReplace(String layerName, int nOut, Distribution dist, WeightInit scheme) {
    if(scheme == WeightInit.DISTRIBUTION) {
        throw new UnsupportedOperationException("Not supported!, Use " +
                "nOutReplace(layerNum, nOut, new WeightInitDistribution(dist), new WeightInitDistribution(distNext)) instead!");
    }
    return nOutReplace(layerName, nOut, new WeightInitDistribution(dist), scheme.getWeightInitFunction());
}
 

@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
                    .list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build();


    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getLayerWiseConfigurations().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getLayerWiseConfigurations().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(bais, true);
    assertEquals(7, net.getLayerWiseConfigurations().getIterationCount());
}
 

@Test
public void testDBNBNMultiLayer() throws Exception {
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    // Run with separate activation layer
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .list()
            .layer(0, new DenseLayer.Builder().nIn(28 * 28).nOut(10).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(1, new BatchNormalization.Builder().nOut(10).build()).layer(2,
                    new ActivationLayer.Builder()
                            .activation(Activation.RELU).build())
            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10)
                    .build())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    network.setInput(next.getFeatures());
    INDArray activationsActual = network.output(next.getFeatures());
    assertEquals(10, activationsActual.shape()[1], 1e-2);

    network.fit(next);
    INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA);
    INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA);
    assertTrue(actualGammaParam != null);
    assertTrue(actualBetaParam != null);
}
 

private static MultiLayerConfiguration getConfCNN(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list()
                    .layer(0, new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1).padding(0, 0)
                                    .activation(Activation.TANH).build())
                    .layer(1, new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1).padding(0, 0)
                                    .activation(Activation.TANH).build())
                    .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10)
                                    .build())
                    .setInputType(InputType.convolutional(10, 10, 3)).build();
    return conf;
}
 

@Test
public void testNeuralNetConfigAPI() {
    LossFunction newLoss = LossFunction.SQUARED_LOSS;
    int newNumIn = numIn + 1;
    int newNumOut = numOut + 1;
    WeightInit newWeight = WeightInit.UNIFORM;
    double newDrop = 0.5;
    int[] newFS = new int[] {3, 3};
    int newFD = 7;
    int[] newStride = new int[] {3, 3};
    Convolution.Type newConvType = Convolution.Type.SAME;
    PoolingType newPoolType = PoolingType.AVG;
    double newCorrupt = 0.5;
    double newSparsity = 0.5;

    MultiLayerConfiguration multiConf1 =
                    new NeuralNetConfiguration.Builder().list()
                                    .layer(0, new DenseLayer.Builder().nIn(newNumIn).nOut(newNumOut).activation(act)
                                                    .build())
                                    .layer(1, new DenseLayer.Builder().nIn(newNumIn + 1).nOut(newNumOut + 1)
                                                    .activation(act).build())
                                    .build();
    NeuralNetConfiguration firstLayer = multiConf1.getConf(0);
    NeuralNetConfiguration secondLayer = multiConf1.getConf(1);

    assertFalse(firstLayer.equals(secondLayer));
}
 

@Test
    public void testTransferVGG() throws Exception {
        //https://github.com/deeplearning4j/deeplearning4j/issues/5167
        DataSet ds = new DataSet();
        ds.setFeatures(Nd4j.create(1, 3, 224, 224));
        ds.setLabels(Nd4j.create(1, 2));

        ComputationGraph model = (ComputationGraph)(
                VGG16.builder().build()
                        .initPretrained(PretrainedType.IMAGENET));
//        System.out.println(model.summary());

        ComputationGraph transferModel = new TransferLearning.GraphBuilder(model)
                .setFeatureExtractor("fc2")
                .removeVertexKeepConnections("predictions")
                .addLayer("predictions",
                        new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                .nIn(4096).nOut(2)
                                .weightInit(WeightInit.XAVIER)
                                .activation(Activation.SOFTMAX).build(), "fc2")
                .build();

//        System.out.println(transferModel.summary());
//        System.out.println("Fitting");
        transferModel.fit(ds);

        ComputationGraph g2 = TestUtils.testModelSerialization(transferModel);
        g2.fit(ds);
    }
 

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(5.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                    new MaxScoreIterationTerminationCondition(10)) //Initial score is ~2.5
                    .scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());

    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}
 

/**
 * Test batchnorm layer.
 */
@Test
public void testConvolutionalLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);
  clf.setNumEpochs(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  ConvolutionLayer convLayer = new ConvolutionLayer();
  convLayer.setKernelSize(new int[]{3, 3});
  convLayer.setActivationFunction(new ActivationReLU());
  convLayer.setNOut(32);
  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);

  clf.setNumEpochs(1);
  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(convLayer, outputLayer);

  clf.buildClassifier(data);
  double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

/**
 * Test batchnorm layer.
 */
@Test
public void testBatchNormLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  data.setClassIndex(data.numAttributes() - 1);
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  DenseLayer dl = new DenseLayer();
  dl.setNOut(8);

  BatchNormalization bn = new BatchNormalization();
  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);
  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(dl, bn, outputLayer);

  clf.setNumEpochs(1);
  clf.buildClassifier(data);
  double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

@Test
public void sanityCheckIssue5662(){
    int mergeSize = 50;
    int encdecSize = 25;
    int in1Size = 20;
    int in2Size = 15;
    int hiddenSize = 10;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .weightInit(WeightInit.XAVIER)
            .graphBuilder()
            .addInputs("in1", "in2")
            .addLayer("1", new DenseLayer.Builder().nOut(mergeSize).build(), "in1")
            .addLayer("2", new DenseLayer.Builder().nOut(mergeSize).build(), "in2")
            .addVertex("merge", new MergeVertex(), "1", "2")
            .addLayer("e",new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(),"merge")
            .addLayer("hidden",new AutoEncoder.Builder().nOut(hiddenSize).build(),"e")
            .addLayer("decoder",new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(),"hidden")
            .addLayer("L4", new DenseLayer.Builder().nOut(mergeSize).build(), "decoder")
            .addLayer("out1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in1Size).build(),"L4")
            .addLayer("out2",new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in2Size).build(),"L4")
            .setOutputs("out1","out2")
            .setInputTypes(InputType.feedForward(in1Size), InputType.feedForward(in2Size))

            .build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(
            new INDArray[]{Nd4j.create(1, in1Size), Nd4j.create(1, in2Size)},
            new INDArray[]{Nd4j.create(1, in1Size), Nd4j.create(1, in2Size)});

    net.summary(InputType.feedForward(in1Size), InputType.feedForward(in2Size));
    net.fit(new SingletonMultiDataSetIterator(mds));
}
 

@Test
public void testTBPTTLongerThanTS() {
    //Extremely simple test of the 'does it throw an exception' variety
    int timeSeriesLength = 20;
    int tbpttLength = 1000;
    int miniBatchSize = 7;
    int nIn = 5;
    int nOut = 4;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
                                    .activation(Activation.TANH).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
                                    .activation(Activation.TANH).build())
                    .layer(2, new RnnOutputLayer.Builder(LossFunction.MSE).nIn(8).nOut(nOut)
                                    .activation(Activation.IDENTITY).build())
                    .backpropType(BackpropType.TruncatedBPTT)
                    .tBPTTBackwardLength(tbpttLength).tBPTTForwardLength(tbpttLength).build();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();

    INDArray features = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength});
    INDArray labels = Nd4j.rand(new int[] {miniBatchSize, nOut, timeSeriesLength});

    INDArray maskArrayInput = Nd4j.ones(miniBatchSize, timeSeriesLength);
    INDArray maskArrayOutput = Nd4j.ones(miniBatchSize, timeSeriesLength);

    DataSet ds = new DataSet(features, labels, maskArrayInput, maskArrayOutput);

    INDArray initialParams = mln.params().dup();
    mln.fit(ds);
    INDArray afterParams = mln.params();
    assertNotEquals(initialParams, afterParams);
}
 

public AttentionVertex build(){
    this.nHeads = nHeads == 0 ? 1 : nHeads;
    this.weightInit = weightInit == null ? WeightInit.XAVIER : weightInit;
    Preconditions.checkArgument(nOut > 0, "You have to set nOut");
    Preconditions.checkArgument(nInKeys > 0, "You have to set nInKeys");
    Preconditions.checkArgument(nInQueries > 0, "You have to set nInQueries");
    Preconditions.checkArgument(nInValues > 0, "You have to set nInValues");
    Preconditions.checkArgument(headSize > 0 || nOut % this.nHeads == 0, "You have to set a head size if nOut isn't cleanly divided by nHeads");
    Preconditions.checkArgument(projectInput || (nInQueries == nInKeys && nInKeys == nInValues  && nInValues == nOut && nHeads == 1), "You may only disable projectInput if all nIn* equal to nOut and you want to use only a single attention head");
    this.headSize = headSize == 0 ? nOut / nHeads : headSize;

    return new AttentionVertex(this);
}
 

private static MultiLayerConfiguration getConf(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build())
                    .build();
    return conf;
}
 

@Test
public void testUpsamplingConvnetJson() {
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true)
            .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
            .layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(new Upsampling2D.Builder().size(2).build())
            .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(new Upsampling2D.Builder().size(2).build())
            .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                    .build())

            .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    MultiLayerConfiguration conf = builder.build();
    String json = conf.toJson();
    MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 

@Test
public void testSerialization() throws IOException, ClassNotFoundException {
  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setSeed(42);
  nnc.setWeightInit(WeightInit.UNIFORM);
  nnc.setL1(5);
  nnc.setL2(5);
  nnc.setUpdater(new AdaMax());
  nnc.setBiasUpdater(new AdaMax());
  nnc.setDropout(new Dropout());
  nnc.setWeightNoise(new DropConnect());
  nnc.setGradientNormalization(GradientNormalization.None);
  nnc.setDist(new weka.dl4j.distribution.ConstantDistribution());

  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  final File output = Paths.get(System.getProperty("java.io.tmpdir"), "nnc.object").toFile();
  ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(output));
  oos.writeObject(nnc);
  oos.close();
  ObjectInputStream ois = new ObjectInputStream(new FileInputStream(output));
  NeuralNetConfiguration nnc2 = (NeuralNetConfiguration) ois.readObject();
  assertEquals(nnc.dist, nnc2.dist);
  assertEquals(nnc.dropout, nnc2.dropout);
  assertEquals(nnc.updater, nnc2.updater);
  assertEquals(nnc.biasUpdater, nnc2.biasUpdater);
  assertEquals(nnc.weightNoise, nnc2.weightNoise);
  assertEquals(nnc, nnc2);
  output.delete();
}
 

@Test
public void testBackProp() {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER)
                                                    .activation(Activation.SOFTMAX).nIn(2).nOut(3).build())
                    .build();


    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.setListeners(new ScoreIterationListener(1));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    DataSet next = iter.next();
    next.normalizeZeroMeanZeroUnitVariance();
    SplitTestAndTrain trainTest = next.splitTestAndTrain(110);
    network.setInput(trainTest.getTrain().getFeatures());
    network.setLabels(trainTest.getTrain().getLabels());
    network.init();
    for( int i=0; i<5; i++ ) {
        network.fit(trainTest.getTrain());
    }

    DataSet test = trainTest.getTest();
    Evaluation eval = new Evaluation();
    INDArray output = network.output(test.getFeatures());
    eval.eval(test.getLabels(), output);
    log.info("Score " + eval.stats());
}
 

@Test
public void testWeightInit() throws Exception {
  List<WeightInit> skipWeightInits = new ArrayList<>();
  skipWeightInits.add(WeightInit.IDENTITY);
  skipWeightInits.add(WeightInit.DISTRIBUTION);
  for (WeightInit wi : WeightInit.values()) {
    if (skipWeightInits.contains(wi)) {
      continue;
    }
    NeuralNetConfiguration conf = new NeuralNetConfiguration();
    conf.setWeightInit(wi);
    checkAppliedParameters(conf, wi.getWeightInitFunction(), BaseLayer::getWeightInitFn);
  }

}
 

@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(5));

    JavaRDD<DataSet> irisData = getIris();

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                                    new ScoreImprovementEpochTerminationCondition(5))
                                    .iterationTerminationConditions(new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5
                                    .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc()))
                                    .modelSaver(saver).build();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new SparkEarlyStoppingTrainer(getContext().sc(),
                    new ParameterAveragingTrainingMaster(true, 4, 1, 150 / 10, 1, 0), esConf, net, irisData);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertTrue(result.getTotalEpochs() < 12); //Normally expect 6 epochs exactly; get a little more than that here due to rounding + order of operations
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 

@Test
public void testEarlyStoppingListeners() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
            .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                    .activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);

    TestListener tl = new TestListener();
    net.setListeners(tl);

    DataSetIterator irisIter = new IrisDataSetIterator(50, 150);
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
            new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                    .iterationTerminationConditions(
                            new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES))
                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                    .build();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new EarlyStoppingTrainer(esConf, net, irisIter);

    EarlyStoppingResult<MultiLayerNetwork> result = trainer.fit();

    assertEquals(5, tl.countEpochStart);
    assertEquals(5, tl.countEpochEnd);
    assertEquals(5 * 150/50, tl.iterCount);

    assertEquals(4, tl.maxEpochStart);
    assertEquals(4, tl.maxEpochEnd);
}
 

@Test
public void testSerialization() throws IOException, ClassNotFoundException {
  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setSeed(42);
  nnc.setWeightInit(WeightInit.UNIFORM);
  nnc.setL1(5);
  nnc.setL2(5);
  nnc.setUpdater(new AdaMax());
  nnc.setBiasUpdater(new AdaMax());
  nnc.setDropout(new Dropout());
  nnc.setWeightNoise(new DropConnect());
  nnc.setGradientNormalization(GradientNormalization.None);
  nnc.setDist(new weka.dl4j.distribution.ConstantDistribution());

  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  final File output = Paths.get(System.getProperty("java.io.tmpdir"), "nnc.object").toFile();
  ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(output));
  oos.writeObject(nnc);
  oos.close();
  ObjectInputStream ois = new ObjectInputStream(new FileInputStream(output));
  NeuralNetConfiguration nnc2 = (NeuralNetConfiguration) ois.readObject();
  assertEquals(nnc.dist, nnc2.dist);
  assertEquals(nnc.dropout, nnc2.dropout);
  assertEquals(nnc.updater, nnc2.updater);
  assertEquals(nnc.biasUpdater, nnc2.biasUpdater);
  assertEquals(nnc.weightNoise, nnc2.weightNoise);
  assertEquals(nnc, nnc2);
  output.delete();
}
 

@Test
public void testEarlyStoppingListenersCG() {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER)
            .graphBuilder()
            .addInputs("in")
            .layer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                    .activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
            .setOutputs("0")
            .build();
    ComputationGraph net = new ComputationGraph(conf);

    TestEarlyStopping.TestListener tl = new TestEarlyStopping.TestListener();
    net.setListeners(tl);

    DataSetIterator irisIter = new IrisDataSetIterator(50, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf =
            new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                    .iterationTerminationConditions(
                            new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES))
                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                    .build();

    IEarlyStoppingTrainer<ComputationGraph> trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);

    trainer.fit();

    assertEquals(5, tl.getCountEpochStart());
    assertEquals(5, tl.getCountEpochEnd());
    assertEquals(5 * 150/50, tl.getIterCount());

    assertEquals(4, tl.getMaxEpochStart());
    assertEquals(4, tl.getMaxEpochEnd());
}