org.deeplearning4j.nn.multilayer.MultiLayerNetwork Java Examples

@Override
protected void postInit() {
    super.postInit();

    if (accumulator == null) {
        log.warn("GradientsAccumulator is undefined, gradients sharing will be skipped");
        return;
    }

    // just pass accumulator down the hill
    if (replicatedModel instanceof ComputationGraph) {
        ((ComputationGraph) replicatedModel).setGradientsAccumulator(accumulator);
    } else if (replicatedModel instanceof MultiLayerNetwork) {
        ((MultiLayerNetwork) replicatedModel).setGradientsAccumulator(accumulator);
    }

    // need to attach this device id to accumulator's workspaces
    accumulator.touch();
}
 

@Test
public void testSequentialLambdaLayerImport() throws Exception {
    KerasLayer.registerLambdaLayer("lambda_1", new ExponentialLambda());
    KerasLayer.registerLambdaLayer("lambda_2", new TimesThreeLambda());

    String modelPath = "modelimport/keras/examples/lambda/sequential_lambda.h5";

    try(InputStream is = Resources.asStream(modelPath)) {
        File modelFile = testDir.newFile("tempModel" + System.currentTimeMillis() + ".h5");
        Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
        MultiLayerNetwork model = new KerasSequentialModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath())
                .enforceTrainingConfig(false).buildSequential().getMultiLayerNetwork();

        System.out.println(model.summary());
        INDArray input = Nd4j.create(new int[]{10, 100});

        model.output(input);
    } finally {
        KerasLayer.clearLambdaLayers();
    }
}
 

@Test
public void testCGEvaluation() {

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration mlnConfig = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlnConfig);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(75, 150);

    net.fit(iris);
    iris.reset();
    graph.fit(iris);

    iris.reset();
    Evaluation evalExpected = net.evaluate(iris);
    iris.reset();
    Evaluation evalActual = graph.evaluate(iris);

    assertEquals(evalExpected.accuracy(), evalActual.accuracy(), 0e-4);
}
 

private static Map<String,INDArray> getFrozenLayerParamCopies(Model m){
    Map<String,INDArray> out = new LinkedHashMap<>();
    org.deeplearning4j.nn.api.Layer[] layers;
    if (m instanceof MultiLayerNetwork) {
        layers = ((MultiLayerNetwork) m).getLayers();
    } else {
        layers = ((ComputationGraph) m).getLayers();
    }

    for(org.deeplearning4j.nn.api.Layer l : layers){
        if(l instanceof FrozenLayer){
            String paramPrefix;
            if(m instanceof MultiLayerNetwork){
                paramPrefix = l.getIndex() + "_";
            } else {
                paramPrefix = l.conf().getLayer().getLayerName() + "_";
            }
            Map<String,INDArray> paramTable = l.paramTable();
            for(Map.Entry<String,INDArray> e : paramTable.entrySet()){
                out.put(paramPrefix + e.getKey(), e.getValue().dup());
            }
        }
    }

    return out;
}
 

private MultiLayerNetwork getConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new ConvolutionLayer.Builder()
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .dataFormat(format)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    } else {
        return getNetWithLayer(new ConvolutionLayer.Builder()
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    }
}
 

@Override
public VariationalAutoencoder getVaeLayer() {
    MultiLayerNetwork network =
                    new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue()));
    network.init();
    INDArray val = ((INDArray) params.value()).unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    Layer l = network.getLayer(0);
    if (!(l instanceof VariationalAutoencoder)) {
        throw new RuntimeException(
                        "Cannot use VaeReconstructionErrorWithKeyFunction on network that doesn't have a VAE "
                                        + "layer as layer 0. Layer type: " + l.getClass());
    }
    return (VariationalAutoencoder) l;
}
 

@Test
public void testConvCacheModeSimple(){

    MultiLayerConfiguration conf1 = getConf(CacheMode.NONE);
    MultiLayerConfiguration conf2 = getConf(CacheMode.DEVICE);

    MultiLayerNetwork net1 = new MultiLayerNetwork(conf1);
    net1.init();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    INDArray in = Nd4j.rand(3, 28*28);
    INDArray labels = TestUtils.randomOneHot(3, 10);

    INDArray out1 = net1.output(in);
    INDArray out2 = net2.output(in);
    assertEquals(out1, out2);

    assertEquals(net1.params(), net2.params());
    net1.fit(in, labels);
    net2.fit(in, labels);
    assertEquals(net1.params(), net2.params());
}
 

@Test
public void testSetGetUpdater2() {
    //Same as above test, except that we are doing setUpdater on a new network
    Nd4j.getRandom().setSeed(12345L);
    double lr = 0.03;
    int nIn = 4;
    int nOut = 8;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(lr,0.6)).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(5)
                                    .updater(org.deeplearning4j.nn.conf.Updater.SGD).build())
                    .layer(1, new DenseLayer.Builder().nIn(5).nOut(6)
                                    .updater(new NoOp()).build())
                    .layer(2, new DenseLayer.Builder().nIn(6).nOut(7)
                                    .updater(org.deeplearning4j.nn.conf.Updater.ADAGRAD).build())
                    .layer(3, new OutputLayer.Builder().nIn(7).nOut(nOut).activation(Activation.SOFTMAX)
                                    .updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Updater newUpdater = UpdaterCreator.getUpdater(net);
    net.setUpdater(newUpdater);
    assertTrue(newUpdater == net.getUpdater()); //Should be identical object
}
 

private MultiLayerNetwork getGlobalPoolingNet(CNN2DFormat format, PoolingType pt, boolean setOnLayerAlso) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new GlobalPoolingLayer.Builder(pt)
                .poolingDimensions(format == CNN2DFormat.NCHW ? new int[]{2,3} : new int[]{1,2})
                .build(), format, ConvolutionMode.Same, null);
    } else {
        return getNetWithLayer(new GlobalPoolingLayer.Builder(pt)
                .build(), format, ConvolutionMode.Same, null);
    }
}
 

@Test
    public void testEmbeddingLayerPreluSimple() {
        Random r = new Random(12345);
        int nExamples = 5;
        INDArray input = Nd4j.zeros(nExamples, 1);
        INDArray labels = Nd4j.zeros(nExamples, 3);
        for (int i = 0; i < nExamples; i++) {
            input.putScalar(i, r.nextInt(4));
            labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
        }

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
                .dataType(DataType.DOUBLE)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
                .list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                .updater(new NoOp()).build())
                .layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
                .layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
                        .weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
                        .updater(new NoOp()).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        if (PRINT_RESULTS) {
            System.out.println("testEmbeddingLayerSimple");
//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        String msg = "testEmbeddingLayerSimple";
        assertTrue(msg, gradOK);
    }
 

@Test
public void testVaeWeightNoise(){

    for(boolean ws : new boolean[]{false, true}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345L)
                .trainingWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                .inferenceWorkspaceMode(ws ? WorkspaceMode.ENABLED : WorkspaceMode.NONE)
                .weightNoise(new WeightNoise(new org.deeplearning4j.nn.conf.distribution.NormalDistribution(0.1, 0.3)))
                .list().layer(0,
                        new VariationalAutoencoder.Builder().nIn(10).nOut(3)
                                .encoderLayerSizes(5).decoderLayerSizes(6)
                                .pzxActivationFunction(Activation.TANH)
                                .reconstructionDistribution(new GaussianReconstructionDistribution())
                                .activation(new ActivationTanH())
                                .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray arr = Nd4j.rand(3, 10);
        net.pretrainLayer(0, arr);

    }


}
 

public static INDArray generateOutput(File inputFile, String modelFilePath) throws IOException, InterruptedException {
    final File modelFile = new File(modelFilePath);
    final MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(modelFile);
    final RecordReader recordReader = generateReader(inputFile);
    //final INDArray array = RecordConverter.toArray(recordReader.next());
    final NormalizerStandardize normalizerStandardize = ModelSerializer.restoreNormalizerFromFile(modelFile);
    //normalizerStandardize.transform(array);
    final DataSetIterator dataSetIterator = new RecordReaderDataSetIterator.Builder(recordReader,1).build();
    normalizerStandardize.fit(dataSetIterator);
    dataSetIterator.setPreProcessor(normalizerStandardize);
    return network.output(dataSetIterator);

}
 

@Test
public void testMultiLayerNetworkFrozenLayerParamsAfterBackprop() {
    Nd4j.getRandom().setSeed(12345);
    DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1));

    MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .weightInit(WeightInit.XAVIER)
            .updater(new Sgd(2))
            .list()
            .layer(new DenseLayer.Builder().nIn(4).nOut(3).build())
            .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                            new DenseLayer.Builder().nIn(3).nOut(4).build()))
            .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                            new DenseLayer.Builder().nIn(4).nOut(2).build()))
            .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(
                            new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(2).nOut(1).build()))
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf1);
    network.init();
    INDArray unfrozenLayerParams = network.getLayer(0).params().dup();
    INDArray frozenLayerParams1 = network.getLayer(1).params().dup();
    INDArray frozenLayerParams2 = network.getLayer(2).params().dup();
    INDArray frozenOutputLayerParams = network.getLayer(3).params().dup();

    for (int i = 0; i < 100; i++) {
        network.fit(randomData);
    }

    assertNotEquals(unfrozenLayerParams, network.getLayer(0).params());
    assertEquals(frozenLayerParams1, network.getLayer(1).params());
    assertEquals(frozenLayerParams2, network.getLayer(2).params());
    assertEquals(frozenOutputLayerParams, network.getLayer(3).params());

}
 

private static void setLearningRate(MultiLayerNetwork net, double newLr, ISchedule lrSchedule) {
    int nLayers = net.getnLayers();
    for (int i = 0; i < nLayers; i++) {
        setLearningRate(net, i, newLr, lrSchedule, false);
    }
    refreshUpdater(net);
}
 

@Test
    public void testLabelProbabilities() throws Exception {
        Nd4j.getRandom().setSeed(42);
        DataSetIterator dataSetIterator = getNormalizedIterator();
        MultiLayerNetwork network = getSingleLayer();
        DataSet next = dataSetIterator.next();
        DataSet filtered = next.filterBy(new int[]{0, 1});
        for (int i = 0; i < 10; i++) {
            network.setEpochCount(i);
            network.getLayerWiseConfigurations().setEpochCount(i);
            network.fit(filtered);
        }

        DataSet anomalies = next.filterBy(new int[] {2});
        INDArray output = network.output(anomalies.getFeatures());
        INDArray normalOutput = network.output(anomalies.getFeatures(),false);
        assertEquals(output.lt(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(),
                normalOutput.eq(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(),1e-1);

//        System.out.println("Labels " + anomalies.getLabels());
//        System.out.println("Anomaly output " + normalOutput);
//        System.out.println(output);

        INDArray normalProbs = network.output(filtered.getFeatures());
        INDArray outputForNormalSamples = network.output(filtered.getFeatures(),false);
        System.out.println("Normal probabilities " + normalProbs);
        System.out.println("Normal raw output " + outputForNormalSamples);

        File tmpFile = new File(testDir.getRoot(),"tmp-file-" + UUID.randomUUID().toString());
        ModelSerializer.writeModel(network,tmpFile,true);
        tmpFile.deleteOnExit();

        MultiLayerNetwork multiLayerNetwork = ModelSerializer.restoreMultiLayerNetwork(tmpFile);
        assertEquals(network.params(),multiLayerNetwork.params());
        assertEquals(network.numParams(),multiLayerNetwork.numParams());

    }
 

@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                                    new ScoreImprovementEpochTerminationCondition(5))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 

@Test
public void testLearningRateByParam() {
    double lr = 0.01;
    double biasLr = 0.02;
    int[] nIns = {4, 3, 3};
    int[] nOuts = {3, 3, 3};
    int oldScore = 1;
    int newScore = 1;
    int iteration = 3;
    INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0])
                                    .updater(new Sgd(lr)).biasUpdater(new Sgd(biasLr)).build())
                    .layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).updater(new Sgd(0.7)).build())
                    .layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).lossFunction(LossFunctions.LossFunction.MSE).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(),
                    new NegativeDefaultStepFunction(), null, net);
    assertEquals(lr, ((Sgd)net.getLayer(0).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4);
    assertEquals(biasLr, ((Sgd)net.getLayer(0).conf().getLayer().getUpdaterByParam("b")).getLearningRate(), 1e-4);
    assertEquals(0.7, ((Sgd)net.getLayer(1).conf().getLayer().getUpdaterByParam("gamma")).getLearningRate(), 1e-4);
    assertEquals(0.3, ((Sgd)net.getLayer(2).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4); //From global LR
    assertEquals(0.3, ((Sgd)net.getLayer(2).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4); //From global LR
}
 

private MultiLayerNetwork getNetWithLayer(Layer layer, CNN2DFormat format, ConvolutionMode cm, InputType inputType) {
    NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder()
            .dataType(this.dataType)
            .seed(12345)
            .convolutionMode(cm)
            .list()
            .layer(new ConvolutionLayer.Builder()
                    .kernelSize(3, 3)
                    .stride(2, 2)
                    .activation(Activation.TANH)
                    .dataFormat(format)
                    .nOut(3)
                    .helperAllowFallback(false)
                    .build())
            .layer(layer)
            .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).build())
            .setInputType(inputType != null ? inputType : InputType.convolutional(12, 12, 3, format));

    if(format == CNN2DFormat.NHWC && !(layer instanceof GlobalPoolingLayer)){
        //Add a preprocessor due to the differences in how NHWC and NCHW activations are flattened
        //DL4J's flattening behaviour matches Keras (hence TF) for import compatibility
        builder.inputPreProcessor(2, new ComposableInputPreProcessor(new NHWCToNCHWPreprocessor(), new CnnToFeedForwardPreProcessor()));
    }

    MultiLayerNetwork net = new MultiLayerNetwork(builder.build());
    net.init();
    return net;
}
 

@Test
public void testL1L2NotSet() {
    // Warning thrown only since some layers may not have l1 or l2
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3))
            .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
            .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
}
 

private INDArray getOutput(Model model, INDArray image) {
	if (model instanceof MultiLayerNetwork) {
		MultiLayerNetwork multiLayerNetwork = (MultiLayerNetwork) model;
		multiLayerNetwork.init();
		return multiLayerNetwork.output(image);
	} else {
		ComputationGraph graph = (ComputationGraph) model;
		graph.init();
		return graph.output(image)[0];
	}
}
 

@Test
public void testTwdFirstLayer() throws Exception {
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4)
                    .updater(new Nesterovs(0.9)).dropOut(0.5)
                    .list().layer(0,
                                    new ConvolutionLayer.Builder(8, 8) //16 filters kernel size 8 stride 4
                                                    .stride(4, 4).nOut(16).dropOut(0.5)
                                                    .activation(Activation.RELU).weightInit(
                                                                    WeightInit.XAVIER)
                                                    .build())
                    .layer(1, new ConvolutionLayer.Builder(4, 4) //32 filters kernel size 4 stride 2
                                    .stride(2, 2).nOut(32).dropOut(0.5).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(2, new DenseLayer.Builder() //fully connected with 256 rectified units
                                    .nOut(256).activation(Activation.RELU).weightInit(WeightInit.XAVIER)
                                    .dropOut(0.5).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer
                                    .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1));

    DataSetIterator iter = new MnistDataSetIterator(10, 10);
    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    DataSet ds = iter.next();
    for( int i=0; i<5; i++ ) {
        network.fit(ds);
    }
}
 

private MultiLayerNetwork getSpaceToDepthNet(CNN2DFormat format, boolean setOnLayerAlso) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new SpaceToDepthLayer.Builder()
                .blocks(2)
                .dataFormat(format)
                .build(), format, ConvolutionMode.Same, null);
    } else {
        return getNetWithLayer(new SpaceToDepthLayer.Builder()
                .blocks(2)
                .build(), format, ConvolutionMode.Same, null);
    }
}
 

private MultiLayerNetwork getDeconv2DNet2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new Deconvolution2D.Builder().nOut(2)
                .activation(Activation.TANH)
                .kernelSize(2,2)
                .stride(2,2)
                .build(), format, cm, null);
    } else {
        return getNetWithLayer(new Deconvolution2D.Builder().nOut(2)
                .activation(Activation.TANH)
                .kernelSize(2,2)
                .stride(2,2)
                .build(), format, cm, null);
    }
}
 

public static List<Double> getPredict(MultiLayerNetwork net, DataSetIterator iterator) {
    List<Double> labels = new LinkedList<>();
    while (iterator.hasNext()) {
        org.nd4j.linalg.dataset.DataSet dataSet = iterator.next();

        INDArray output = net.output(dataSet.getFeatures());

        long[] shape = output.shape();
        for (int i = 0; i < shape[0]; i++) {
            labels.add(output.getDouble(i));
        }
    }
    iterator.reset();

    return labels;
}
 

private MultiLayerNetwork getGlobalPoolingNet(CNN2DFormat format, PoolingType pt, boolean setOnLayerAlso) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new GlobalPoolingLayer.Builder(pt)
                .poolingDimensions(format == CNN2DFormat.NCHW ? new int[]{2,3} : new int[]{1,2})
                .build(), format, ConvolutionMode.Same, null);
    } else {
        return getNetWithLayer(new GlobalPoolingLayer.Builder(pt)
                .build(), format, ConvolutionMode.Same, null);
    }
}
 

@Test
public void testEmbeddingSequenceLayerConfig() {

    int inputLength = 6;
    int nIn = 10;
    int embeddingDim = 5;
    int nout = 4;

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(new EmbeddingSequenceLayer.Builder().hasBias(hasBias)
                        .inputLength(inputLength).nIn(nIn).nOut(embeddingDim).build())
                .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nout).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingSequenceLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Test
public void testDenseBatchNorm(){


    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .dataType(DataType.DOUBLE)
            .seed(12345)
            .weightInit(WeightInit.XAVIER)
            .updater(new NoOp())
            .list()
            .layer(new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build())
            .layer(new BatchNormalization.Builder().nOut(5).build())
            .layer(new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray in = Nd4j.rand(3, 5);
    INDArray labels = TestUtils.randomOneHot(3, 5);

    //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
    //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
    //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
    Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, in, labels, null, null, false, -1, excludeParams, null);

    assertTrue(gradOK);

    TestUtils.testModelSerialization(net);
}
 

private void updateExamplesMinibatchesCounts(Model model) {
    ModelInfo modelInfo = getModelInfo(model);
    int examplesThisMinibatch = 0;
    if (model instanceof MultiLayerNetwork) {
        examplesThisMinibatch = ((MultiLayerNetwork) model).batchSize();
    } else if (model instanceof ComputationGraph) {
        examplesThisMinibatch = ((ComputationGraph) model).batchSize();
    } else if (model instanceof Layer) {
        examplesThisMinibatch = ((Layer) model).getInputMiniBatchSize();
    }
    modelInfo.examplesSinceLastReport += examplesThisMinibatch;
    modelInfo.totalExamples += examplesThisMinibatch;
    modelInfo.minibatchesSinceLastReport++;
    modelInfo.totalMinibatches++;
}
 

@Override
public double calculateScore(MultiLayerNetwork network) {
    dataSetIterator.reset();

    double losSum = 0.0;
    int exCount = 0;
    while (dataSetIterator.hasNext()) {
        DataSet dataSet = dataSetIterator.next();
        if (dataSet == null) {
            break;
        }
        long nEx = dataSet.getFeatures().size(0);

        INDArray output = network.output(dataSet.getFeatures(), false);
        INDArray labels = dataSet.getLabels();

        INDArray score = Transforms.abs(output.sub(labels));
        score = score.div(labels);

        exCount += nEx;
        losSum += score.sumNumber().doubleValue();
    }

    if (average) {
        return losSum / exCount;
    }
    return losSum;
}
 

protected static int getEpoch(Model model) {
    if (model instanceof MultiLayerNetwork) {
        return ((MultiLayerNetwork) model).getLayerWiseConfigurations().getEpochCount();
    } else if (model instanceof ComputationGraph) {
        return ((ComputationGraph) model).getConfiguration().getEpochCount();
    } else {
        return model.conf().getEpochCount();
    }
}