org.deeplearning4j.nn.api.OptimizationAlgorithm Java Examples

@Test
public void testRnnTimeStepWithPreprocessor() {

    MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder()
                                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                                    .list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10)
                                                    .nOut(10).activation(Activation.TANH).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(10)
                                                    .nOut(10).activation(Activation.TANH).build())
                                    .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                    .activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
                                    .inputPreProcessor(0, new FeedForwardToRnnPreProcessor())
                                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray in = Nd4j.rand(1, 10);
    net.rnnTimeStep(in);
}
 

protected static ComputationGraphConfiguration getMultiInputGraphConfig() {
    ComputationGraphConfiguration conf =
                    new NeuralNetConfiguration.Builder()
                                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                                    .graphBuilder().addInputs("input")
                                    .setInputTypes(InputType.convolutional(32, 32, 3))
                                    .addLayer("cnn1",
                                                    new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3)
                                                                    .build(),
                                                    "input")
                                    .addLayer("cnn2",
                                                    new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3)
                                                                    .build(),
                                                    "input")
                                    .addLayer("max1",
                                                    new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                                                    .stride(1, 1).kernelSize(2, 2).build(),
                                                    "cnn1", "cnn2")
                                    .addLayer("dnn1", new DenseLayer.Builder().nOut(7).build(), "max1")
                                    .addLayer("output", new OutputLayer.Builder().nIn(7).nOut(10).activation(Activation.SOFTMAX).build(), "dnn1")
                                    .setOutputs("output").build();

    return conf;
}
 

private static MultiLayerNetwork softMaxRegression(int seed,
		int iterations, int numRows, int numColumns, int outputNum) {
	MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
			.seed(seed)
			.gradientNormalization(
					GradientNormalization.ClipElementWiseAbsoluteValue)
			.gradientNormalizationThreshold(1.0)
			.iterations(iterations)
			.momentum(0.5)
			.momentumAfter(Collections.singletonMap(3, 0.9))
			.optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
			.list(1)
			.layer(0,
					new OutputLayer.Builder(
							LossFunction.NEGATIVELOGLIKELIHOOD)
							.activation("softmax")
							.nIn(numColumns * numRows).nOut(outputNum)
							.build()).pretrain(true).backprop(false)
			.build();

	MultiLayerNetwork model = new MultiLayerNetwork(conf);

	return model;
}
 

private ComputationGraph getGraph(int numLabels, double lambda) {
    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .dist(new NormalDistribution(0, 1)).updater(new NoOp())
                    .graphBuilder().addInputs("input1")
                    .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(),
                                    "input1")
                    .addLayer("lossLayer", new CenterLossOutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels)
                                    .lambda(lambda).activation(Activation.SOFTMAX).build(), "l1")
                    .setOutputs("lossLayer").build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    return graph;
}
 

public MultiLayerConfiguration mnistLenet() {
    MultiLayerConfiguration builder =
                    new NeuralNetConfiguration.Builder().seed(3)
                                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nIn(1).nOut(6).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {5, 5}, new int[] {2, 2}).build())
                                    .layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nIn(1).nOut(6).build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {5, 5}, new int[] {2, 2}).build())
                                    .layer(4, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                                    LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(150)
                                                                    .nOut(10).build())
                                    .build();
    return builder;
}
 

@Test
public void testSetParams() {
    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
                    .updater(new Sgd(1e-1))
                    .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(4).nOut(3)
                                    .weightInit(WeightInit.ZERO).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();

    long numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    OutputLayer l = (OutputLayer) conf.getLayer().instantiate(conf,
                    Collections.<TrainingListener>singletonList(new ScoreIterationListener(1)), 0, params, true, params.dataType());
    params = l.params();
    l.setParams(params);
    assertEquals(params, l.params());
}
 

@Test
public void testMultiCNNLayer() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new LocalResponseNormalization.Builder().build()).layer(2,
                                    new DenseLayer.Builder()
                                                    .nOut(2).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10)
                                    .build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    network.fit(next);
}
 

@Test
public void testConjugateGradientDescent() throws Exception {
  DenseLayer dl1 = new DenseLayer();
  dl1.setNOut(16);

  OutputLayer ol = new OutputLayer();

  Layer[] ls = new Layer[]{dl1, ol};

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT);
  nnc.setSeed(TestUtil.SEED);
  clf.setNeuralNetConfiguration(nnc);

  clf.setNumEpochs(3);
  clf.setLayers(ls);
  final EarlyStopping config = new EarlyStopping(0, 0);
  clf.setEarlyStopping(config);
  TestUtil.crossValidate(clf, DatasetLoader.loadGlass());
}
 

@Test
public void testTruncatedBPTTSimple() {
    //Extremely simple test of the 'does it throw an exception' variety
    int timeSeriesLength = 12;
    int miniBatchSize = 7;
    int nIn = 5;
    int nOut = 4;

    int nTimeSlices = 20;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
                                    .activation(Activation.TANH)
                                    .dist(new NormalDistribution(0, 0.5)).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
                                    .activation(Activation.TANH)
                                    .dist(
                                                    new NormalDistribution(0,
                                                                    0.5))
                                    .build())
                    .layer(2, new RnnOutputLayer.Builder(LossFunction.MCXENT)
                                    .nIn(8).nOut(nOut).activation(Activation.SOFTMAX)
                                    .dist(new NormalDistribution(0, 0.5))
                                    .build())
                    .backpropType(BackpropType.TruncatedBPTT)
                    .tBPTTBackwardLength(timeSeriesLength).tBPTTForwardLength(timeSeriesLength).build();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
    mln.init();

    INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, nTimeSlices * timeSeriesLength});
    INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, nTimeSlices * timeSeriesLength});

    mln.fit(inputLong, labelsLong);
}
 

@Test
public void testUpsamplingConvnetJson() {
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true)
            .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
            .layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(new Upsampling2D.Builder().size(2).build())
            .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(new Upsampling2D.Builder().size(2).build())
            .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                    .build())

            .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    MultiLayerConfiguration conf = builder.build();
    String json = conf.toJson();
    MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 

@Test
public void testSetCacheMode() throws Exception {
  clf.setInstanceIterator(idiMnist);

  DenseLayer denseLayer = new DenseLayer();
  denseLayer.setNOut(128);
  denseLayer.setActivationFunction(new ActivationReLU());

  DenseLayer denseLayer2 = new DenseLayer();
  denseLayer2.setNOut(32);
  denseLayer2.setActivationFunction(new ActivationReLU());

  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());
  outputLayer.setLossFn(new LossMCXENT());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setSeed(TestUtil.SEED);

  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(denseLayer, denseLayer2, outputLayer);
  clf.setIterationListener(new EpochListener());

  clf.setCacheMode(CacheMode.FILESYSTEM);
  TestUtil.holdout(clf, dataMnist);
}
 

@Test
public void testEvaluativeListenerSimple(){
    //Sanity check: https://github.com/deeplearning4j/deeplearning4j/issues/5351

    // Network config
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()

            .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42)
            .updater(new Sgd(1e-6)).list()
            .layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build())
            .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                    LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.SOFTMAX).build())
            .build();

    // Instantiate model
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    // Train-test split
    DataSetIterator iter = new IrisDataSetIterator(30, 150);
    DataSetIterator iterTest = new IrisDataSetIterator(30, 150);

    net.setListeners(new EvaluativeListener(iterTest, 3));

    for( int i=0; i<3; i++ ){
        net.fit(iter);
    }
}
 

@Test
public void testSerialization() throws IOException, ClassNotFoundException {
  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setSeed(42);
  nnc.setWeightInit(WeightInit.UNIFORM);
  nnc.setL1(5);
  nnc.setL2(5);
  nnc.setUpdater(new AdaMax());
  nnc.setBiasUpdater(new AdaMax());
  nnc.setDropout(new Dropout());
  nnc.setWeightNoise(new DropConnect());
  nnc.setGradientNormalization(GradientNormalization.None);
  nnc.setDist(new weka.dl4j.distribution.ConstantDistribution());

  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  final File output = Paths.get(System.getProperty("java.io.tmpdir"), "nnc.object").toFile();
  ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(output));
  oos.writeObject(nnc);
  oos.close();
  ObjectInputStream ois = new ObjectInputStream(new FileInputStream(output));
  NeuralNetConfiguration nnc2 = (NeuralNetConfiguration) ois.readObject();
  assertEquals(nnc.dist, nnc2.dist);
  assertEquals(nnc.dropout, nnc2.dropout);
  assertEquals(nnc.updater, nnc2.updater);
  assertEquals(nnc.biasUpdater, nnc2.biasUpdater);
  assertEquals(nnc.weightNoise, nnc2.weightNoise);
  assertEquals(nnc, nnc2);
  output.delete();
}
 

@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                                    new ScoreImprovementEpochTerminationCondition(5))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 

@Test
public void testConvnetJson() {
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l1(1e-1).l2(2e-4).weightNoise(new DropConnect(0.5)).miniBatch(true)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                    .build())

                    .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    MultiLayerConfiguration conf = builder.build();
    String json = conf.toJson();
    MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 

/**
 * Test subsampling layer.
 */
@Test
public void testSubsamplingLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  data.setClassIndex(data.numAttributes() - 1);
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  SubsamplingLayer pool = new SubsamplingLayer();
  pool.setKernelSizeX(2);
  pool.setKernelSizeY(2);
  pool.setPoolingType(PoolingType.MAX);

  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);

  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(pool, outputLayer);

  clf.setNumEpochs(1);
  clf.buildClassifier(data);
  final double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

/**
 * Test batchnorm layer.
 */
@Test
public void testBatchNormLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  data.setClassIndex(data.numAttributes() - 1);
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  DenseLayer dl = new DenseLayer();
  dl.setNOut(8);

  BatchNormalization bn = new BatchNormalization();
  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);
  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(dl, bn, outputLayer);

  clf.setNumEpochs(1);
  clf.buildClassifier(data);
  double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

/**
 * Test batchnorm layer.
 */
@Test
public void testConvolutionalLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);
  clf.setNumEpochs(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  ConvolutionLayer convLayer = new ConvolutionLayer();
  convLayer.setKernelSize(new int[]{3, 3});
  convLayer.setActivationFunction(new ActivationReLU());
  convLayer.setNOut(32);
  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);

  clf.setNumEpochs(1);
  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(convLayer, outputLayer);

  clf.buildClassifier(data);
  double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

@Test
public void testEarlyStoppingGetBestModel() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.001)).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    MultipleEpochsIterator mIter = new MultipleEpochsIterator(10, irisIter);

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES))
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new EarlyStoppingTrainer(esConf, net, mIter);

    EarlyStoppingResult<MultiLayerNetwork> result = trainer.fit();
    System.out.println(result);

    MultiLayerNetwork mln = result.getBestModel();

    assertEquals(net.getnLayers(), mln.getnLayers());
    assertEquals(net.conf().getOptimizationAlgo(), mln.conf().getOptimizationAlgo());
    BaseLayer bl = (BaseLayer) net.conf().getLayer();
    assertEquals(bl.getActivationFn().toString(), ((BaseLayer) mln.conf().getLayer()).getActivationFn().toString());
    assertEquals(bl.getIUpdater(), ((BaseLayer) mln.conf().getLayer()).getIUpdater());
}
 

@Test
    public void testEmbeddingLayerPreluSimple() {
        Random r = new Random(12345);
        int nExamples = 5;
        INDArray input = Nd4j.zeros(nExamples, 1);
        INDArray labels = Nd4j.zeros(nExamples, 3);
        for (int i = 0; i < nExamples; i++) {
            input.putScalar(i, r.nextInt(4));
            labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
        }

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
                .dataType(DataType.DOUBLE)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
                .list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                .updater(new NoOp()).build())
                .layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
                .layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
                        .weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
                        .updater(new NoOp()).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        if (PRINT_RESULTS) {
            System.out.println("testEmbeddingLayerSimple");
//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        String msg = "testEmbeddingLayerSimple";
        assertTrue(msg, gradOK);
    }
 

/**
 * Test to serialization of the classifier. This is important for the GUI usage.
 *
 * @throws Exception Could not build classifier.
 */
@Test
public void testSerialization() throws Exception {
  clf.setInstanceIterator(idiMnist);

  DenseLayer denseLayer = new DenseLayer();
  denseLayer.setNOut(8);
  denseLayer.setLayerName("Dense-layer");
  denseLayer.setActivationFunction(new ActivationReLU());

  DenseLayer denseLayer2 = new DenseLayer();
  denseLayer2.setNOut(4);
  denseLayer2.setLayerName("Dense-layer");
  denseLayer2.setActivationFunction(new ActivationReLU());

  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());
  outputLayer.setLossFn(new LossMCXENT());
  outputLayer.setLayerName("Output-layer");

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setSeed(TestUtil.SEED);

  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(denseLayer, denseLayer2, outputLayer);
  clf.setIterationListener(new EpochListener());

  File out = Paths.get(System.getProperty("java.io.tmpdir"), "out.object").toFile();
  ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(out));
  clf.initializeClassifier(dataMnist);
  oos.writeObject(clf);

  ObjectInputStream ois = new ObjectInputStream(new FileInputStream(out));
  Dl4jMlpClassifier clf2 = (Dl4jMlpClassifier) ois.readObject();

  clf2.setNumEpochs(1);
  clf2.buildClassifier(dataMnist);
}
 

@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .graphBuilder().addInputs("in")
            .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build(), "in")
            .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                            LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                            .build(),
                    "0")
            .setOutputs("1").build();


    ComputationGraph network = new ComputationGraph(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getConfiguration().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getConfiguration().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    ComputationGraph net = ModelSerializer.restoreComputationGraph(bais, true);
    assertEquals(7, net.getConfiguration().getIterationCount());
}
 

@Test
public void testTimeTermination() {
    //test termination after max time

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(1e-6)).weightInit(WeightInit.XAVIER).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(10000))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(5, TimeUnit.SECONDS),
                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                    .scoreCalculator(new DataSetLossCalculator(irisIter, true))
                    .modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    long startTime = System.currentTimeMillis();
    EarlyStoppingResult result = trainer.fit();
    long endTime = System.currentTimeMillis();
    int durationSeconds = (int) (endTime - startTime) / 1000;

    assertTrue(durationSeconds >= 3);
    assertTrue(durationSeconds <= 20);

    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxTimeIterationTerminationCondition(5, TimeUnit.SECONDS).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 

/**
 * We need to create and set the fine tuning config
 * @return Default fine tuning config
 */
protected FineTuneConfiguration getFineTuneConfig() {
    return new FineTuneConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(new Nesterovs(5e-5))
            .seed(seed)
            .build();
}
 

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(5.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                    new MaxScoreIterationTerminationCondition(10)) //Initial score is ~2.5
                    .scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());

    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}
 

@Test
public void testOptimizationAlgo() throws Exception {
  for (OptimizationAlgorithm optAlgo : OptimizationAlgorithm.values()) {
    NeuralNetConfiguration conf = new NeuralNetConfiguration();
    conf.setOptimizationAlgo(optAlgo);
    log.info(optAlgo.toString());
    final Dl4jMlpClassifier clf = setupClf(conf);
    final OptimizationAlgorithm actual = clf.getModel().conf().getOptimizationAlgo();
    if (!actual.equals(optAlgo)) {
      failMessage.append(String.format("actual=%s,expected=%s", actual, optAlgo));
    }
  }
}
 

/**
 * Test batchnorm layer.
 */
@Test
public void testBatchNormLayer() throws Exception {
  // CLF
  Dl4jMlpClassifier clf = new Dl4jMlpClassifier();
  clf.setSeed(1);

  // Data
  Instances data = DatasetLoader.loadMiniMnistMeta();
  data.setClassIndex(data.numAttributes() - 1);
  final ImageInstanceIterator imgIter = DatasetLoader.loadMiniMnistImageIterator();
  clf.setInstanceIterator(imgIter);

  DenseLayer dl = new DenseLayer();
  dl.setNOut(8);

  BatchNormalization bn = new BatchNormalization();
  OutputLayer outputLayer = new OutputLayer();
  outputLayer.setActivationFunction(new ActivationSoftmax());

  NeuralNetConfiguration nnc = new NeuralNetConfiguration();
  nnc.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
  nnc.setWeightInit(WeightInit.XAVIER);
  clf.setNeuralNetConfiguration(nnc);
  clf.setLayers(dl, bn, outputLayer);

  clf.setNumEpochs(1);
  clf.buildClassifier(data);
  double[][] res = clf.distributionsForInstances(data);
  Assert.assertEquals(DatasetLoader.NUM_INSTANCES_MNIST, res.length);
  Assert.assertEquals(DatasetLoader.NUM_CLASSES_MNIST, res[0].length);
}
 

@Test
public void testBatchNorm() {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(2, new BatchNormalization.Builder().nOut(2).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER)
                                                    .activation(Activation.SOFTMAX).nIn(2).nOut(3).build())
                    .build();


    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.setListeners(new ScoreIterationListener(1));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    DataSet next = iter.next();
    next.normalizeZeroMeanZeroUnitVariance();
    SplitTestAndTrain trainTest = next.splitTestAndTrain(110);
    network.setLabels(trainTest.getTrain().getLabels());
    network.init();
    for( int i=0; i<5; i++ ) {
        network.fit(trainTest.getTrain());
    }

}
 

@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(5.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(10)) //Initial score is ~2.5
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());

    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}
 

private static MultiLayerConfiguration getConf(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build())
                    .build();
    return conf;
}