Java Code Examples for org.deeplearning4j.nn.multilayer.MultiLayerNetwork#getLayer()

@Override
public VariationalAutoencoder getVaeLayer() {
    MultiLayerNetwork network =
                    new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue()));
    network.init();
    INDArray val = ((INDArray) params.value()).unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    Layer l = network.getLayer(0);
    if (!(l instanceof VariationalAutoencoder)) {
        throw new RuntimeException(
                        "Cannot use VaeReconstructionProbWithKeyFunction on network that doesn't have a VAE "
                                        + "layer as layer 0. Layer type: " + l.getClass());
    }
    return (VariationalAutoencoder) l;
}
 

@Test
public void testEmbeddingLayerConfig() {

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(0, new EmbeddingLayer.Builder().hasBias(hasBias).nIn(10).nOut(5).build())
                .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Override
public VariationalAutoencoder getVaeLayer() {
    MultiLayerNetwork network =
                    new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue()));
    network.init();
    INDArray val = ((INDArray) params.value()).unsafeDuplication();
    if (val.length() != network.numParams(false))
        throw new IllegalStateException(
                        "Network did not have same number of parameters as the broadcast set parameters");
    network.setParameters(val);

    Layer l = network.getLayer(0);
    if (!(l instanceof VariationalAutoencoder)) {
        throw new RuntimeException(
                        "Cannot use VaeReconstructionErrorWithKeyFunction on network that doesn't have a VAE "
                                        + "layer as layer 0. Layer type: " + l.getClass());
    }
    return (VariationalAutoencoder) l;
}
 

@Test
public void testEmbeddingSequenceLayerConfig() {

    int inputLength = 6;
    int nIn = 10;
    int embeddingDim = 5;
    int nout = 4;

    for (boolean hasBias : new boolean[]{true, false}) {
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list()
                .layer(new EmbeddingSequenceLayer.Builder().hasBias(hasBias)
                        .inputLength(inputLength).nIn(nIn).nOut(embeddingDim).build())
                .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nout).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l0 = net.getLayer(0);

        assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingSequenceLayer.class, l0.getClass());
        assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn());
        assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut());

        INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY);
        INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY);
        assertArrayEquals(new long[]{10, 5}, weights.shape());
        if (hasBias) {
            assertArrayEquals(new long[]{1, 5}, bias.shape());
        }
    }
}
 

@Test
public void testDropConnectValues() {
    Nd4j.getRandom().setSeed(12345);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .weightInit(WeightInit.ONES)
            .list()
            .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Layer l = net.getLayer(0);
    DropConnect d = new DropConnect(0.5);

    INDArray outTest = d.getParameter(l, "W", 0, 0, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(l.getParam("W") == outTest);    //Should be same object
    INDArray outTrain = d.getParameter(l, "W", 0, 0, true, LayerWorkspaceMgr.noWorkspaces());
    assertNotEquals(l.getParam("W"), outTrain);

    assertEquals(l.getParam("W"), Nd4j.ones(DataType.FLOAT, 10, 10));

    int countZeros = Nd4j.getExecutioner().exec(new MatchCondition(outTrain, Conditions.equals(0))).getInt(0);
    int countOnes = Nd4j.getExecutioner().exec(new MatchCondition(outTrain, Conditions.equals(1))).getInt(0);

    assertEquals(100, countZeros + countOnes);  //Should only be 0 or 2
    //Stochastic, but this should hold for most cases
    assertTrue(countZeros >= 25 && countZeros <= 75);
    assertTrue(countOnes >= 25 && countOnes <= 75);
}
 

@Test
public void testCausalCon1D() throws Exception {
    String[] names = new String[]{
            "causal_conv1d_k2_s1_d1_cl_model.h5",
            "causal_conv1d_k2_s1_d2_cl_model.h5",
            "causal_conv1d_k2_s2_d1_cl_model.h5",
            "causal_conv1d_k2_s3_d1_cl_model.h5",
            "causal_conv1d_k3_s1_d1_cl_model.h5",
            "causal_conv1d_k3_s1_d2_cl_model.h5",
            "causal_conv1d_k3_s2_d1_cl_model.h5",
            "causal_conv1d_k3_s3_d1_cl_model.h5",
            "causal_conv1d_k4_s1_d1_cl_model.h5",
            "causal_conv1d_k4_s1_d2_cl_model.h5",
            "causal_conv1d_k4_s2_d1_cl_model.h5",
            "causal_conv1d_k4_s3_d1_cl_model.h5"
    };

    for(String name : names ){
        System.out.println("Starting test: " + name);
        String modelPath = "modelimport/keras/examples/causal_conv1d/" + name;
        String inputsOutputPath = "modelimport/keras/examples/causal_conv1d/" + (name.substring(0,name.length()-"model.h5".length()) + "inputs_and_outputs.h5");

        MultiLayerNetwork net = importEndModelTest(modelPath, inputsOutputPath, true, true,
                true, true, false, null, null);
        Layer l = net.getLayer(0);
        Convolution1DLayer c1d = (Convolution1DLayer) l.getConfig();
        assertEquals(ConvolutionMode.Causal, c1d.getConvolutionMode());
    }
}
 

@Test
public void testConvolutionalNoBias() throws Exception {
    int[] minibatchSizes = {1, 4};
    int width = 6;
    int height = 6;
    int inputDepth = 2;
    int nOut = 3;

    Field f = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper");
    f.setAccessible(true);

    Random r = new Random(12345);
    for (int minibatchSize : minibatchSizes) {
        for (boolean convHasBias : new boolean[]{true, false}) {

            INDArray input = Nd4j.rand(new int[]{minibatchSize, inputDepth, height, width});
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(i, r.nextInt(nOut), 1.0);
            }

            MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                    .dataType(DataType.DOUBLE)
                    .dist(new UniformDistribution(-1, 1))
                    .updater(new NoOp()).seed(12345L)
                    .list()
                    .layer(0, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(1, 1).nOut(3)
                            .hasBias(convHasBias)
                            .activation(Activation.TANH).build())
                    .layer(1, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(0, 0).nOut(3)
                            .hasBias(convHasBias)
                            .activation(Activation.TANH).build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutional(height, width, inputDepth))
                    ;

            MultiLayerConfiguration conf = builder.build();

            MultiLayerNetwork mln = new MultiLayerNetwork(conf);
            mln.init();

            org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c0 =
                    (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(0);
            ConvolutionHelper ch0 = (ConvolutionHelper) f.get(c0);
            assertTrue(ch0 instanceof CudnnConvolutionHelper);

            org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c1 =
                    (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(1);
            ConvolutionHelper ch1 = (ConvolutionHelper) f.get(c1);
            assertTrue(ch1 instanceof CudnnConvolutionHelper);


            String name = new Object() {}.getClass().getEnclosingMethod().getName() + ", minibatch = "
                    + minibatchSize + ", convHasBias = " + convHasBias;

            if (PRINT_RESULTS) {
                System.out.println(name);
                for (int j = 0; j < mln.getnLayers(); j++)
                    System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

            assertTrue(name, gradOK);
        }
    }
}
 

@Test
public void validateImplMultiLayerTBPTT() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int inputSize = 3;
    int lstmLayerSize = 4;
    int timeSeriesLength = 23;
    int tbpttLength = 5;
    int nOut = 2;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new LSTM.Builder().nIn(inputSize).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build())
                    .backpropType(BackpropType.TruncatedBPTT)
                    .tBPTTLength(tbpttLength).build();

    MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone());
    mln1.init();

    MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone());
    mln2.init();


    assertEquals(mln1.params(), mln2.params());

    Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper");
    f.setAccessible(true);

    Layer l0 = mln1.getLayer(0);
    Layer l1 = mln1.getLayer(1);
    f.set(l0, null);
    f.set(l1, null);
    assertNull(f.get(l0));
    assertNull(f.get(l1));

    l0 = mln2.getLayer(0);
    l1 = mln2.getLayer(1);
    assertTrue(f.get(l0) instanceof CudnnLSTMHelper);
    assertTrue(f.get(l1) instanceof CudnnLSTMHelper);

    Random r = new Random(12345);
    for (int x = 0; x < 1; x++) {
        INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength});
        INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength);
        for (int i = 0; i < minibatch; i++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                labels.putScalar(i, r.nextInt(nOut), j, 1.0);
            }
        }

        DataSet ds = new DataSet(input, labels);
        mln1.fit(ds);
        mln2.fit(ds);
    }


    assertEquals(mln1.params(), mln2.params());
}
 

@Test
public void validateImplMultiLayer() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int inputSize = 3;
    int lstmLayerSize = 4;
    int timeSeriesLength = 3;
    int nOut = 2;
    INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength});
    INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        for (int j = 0; j < timeSeriesLength; j++) {
            labels.putScalar(i, r.nextInt(nOut), j, 1.0);
        }
    }

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build())
                    .build();

    MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone());
    mln1.init();

    MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone());
    mln2.init();


    assertEquals(mln1.params(), mln2.params());

    Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper");
    f.setAccessible(true);

    Layer l0 = mln1.getLayer(0);
    Layer l1 = mln1.getLayer(1);
    f.set(l0, null);
    f.set(l1, null);
    assertNull(f.get(l0));
    assertNull(f.get(l1));

    l0 = mln2.getLayer(0);
    l1 = mln2.getLayer(1);
    assertTrue(f.get(l0) instanceof CudnnLSTMHelper);
    assertTrue(f.get(l1) instanceof CudnnLSTMHelper);


    INDArray out1 = mln1.output(input);
    INDArray out2 = mln2.output(input);

    assertEquals(out1, out2);

    for (int x = 0; x < 10; x++) {
        input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength});
        labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength);
        for (int i = 0; i < minibatch; i++) {
            for (int j = 0; j < timeSeriesLength; j++) {
                labels.putScalar(i, r.nextInt(nOut), j, 1.0);
            }
        }

        mln1.setInput(input);
        mln1.setLabels(labels);

        mln2.setInput(input);
        mln2.setLabels(labels);

        mln1.computeGradientAndScore();
        mln2.computeGradientAndScore();

        assertEquals(mln1.score(), mln2.score(), 1e-5);

        assertEquals(mln1.getFlattenedGradients(), mln2.getFlattenedGradients());

        mln1.fit(new DataSet(input, labels));
        mln2.fit(new DataSet(input, labels));

        assertEquals("Iteration: " + x, mln1.params(), mln2.params());
    }
}
 

@Test
public void validateImplSimple() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int inputSize = 3;
    int lstmLayerSize = 4;
    int timeSeriesLength = 3;
    int nOut = 2;
    INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength});
    INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        for (int j = 0; j < timeSeriesLength; j++) {
            labels.putScalar(i, r.nextInt(nOut), j, 1.0);
        }
    }

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().inferenceWorkspaceMode(WorkspaceMode.NONE)
                    .trainingWorkspaceMode(WorkspaceMode.NONE).updater(new NoOp())
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build())
                    .build();

    MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone());
    mln1.init();

    MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone());
    mln2.init();


    assertEquals(mln1.params(), mln2.params());

    Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper");
    f.setAccessible(true);

    Layer l0 = mln1.getLayer(0);
    f.set(l0, null);
    assertNull(f.get(l0));

    l0 = mln2.getLayer(0);
    assertTrue(f.get(l0) instanceof CudnnLSTMHelper);


    INDArray out1 = mln1.output(input);
    INDArray out2 = mln2.output(input);

    assertEquals(out1, out2);


    mln1.setInput(input);
    mln1.setLabels(labels);

    mln2.setInput(input);
    mln2.setLabels(labels);

    mln1.computeGradientAndScore();
    mln2.computeGradientAndScore();

    assertEquals(mln1.score(), mln2.score(), 1e-5);

    Gradient g1 = mln1.gradient();
    Gradient g2 = mln2.gradient();

    for (Map.Entry<String, INDArray> entry : g1.gradientForVariable().entrySet()) {
        INDArray exp = entry.getValue();
        INDArray act = g2.gradientForVariable().get(entry.getKey());

        //System.out.println(entry.getKey() + "\t" + exp.equals(act));
    }

    assertEquals(mln1.getFlattenedGradients(), mln2.getFlattenedGradients());
}
 

@Test
public void testCropping2DLayer() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 2;

    int[] minibatchSizes = {1, 3};
    int width = 12;
    int height = 11;
    int[] inputDepths = {1, 3};

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};

    int[][] cropTestCases = new int[][]{{0, 0, 0, 0}, {1, 1, 0, 0}, {2, 2, 2, 2}, {1, 2, 3, 4}};

    for (int inputDepth : inputDepths) {
        for (int minibatchSize : minibatchSizes) {
            INDArray input = Nd4j.rand(new int[]{minibatchSize, inputDepth, height, width});
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }
            for (int[] crop : cropTestCases) {

                MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder()
                                .dataType(DataType.DOUBLE)
                                .updater(new NoOp())
                                .convolutionMode(ConvolutionMode.Same)
                                .weightInit(new NormalDistribution(0, 1)).list()
                                .layer(new ConvolutionLayer.Builder(kernel, stride, padding)
                                        .nIn(inputDepth).nOut(2).build())//output: (6-2+0)/1+1 = 5
                                .layer(new Cropping2D(crop))
                                .layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(2).nOut(2).build())
                                .layer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG).kernelSize(3, 3).stride(3, 3).build())
                                .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nOut(nOut).build())
                                .setInputType(InputType.convolutional(height, width, inputDepth))
                                .build();

                MultiLayerNetwork net = new MultiLayerNetwork(conf);
                net.init();

                //Check cropping activation shape
                org.deeplearning4j.nn.layers.convolution.Cropping2DLayer cl =
                        (org.deeplearning4j.nn.layers.convolution.Cropping2DLayer) net.getLayer(1);
                val expShape = new long[]{minibatchSize, inputDepth, height - crop[0] - crop[1],
                        width - crop[2] - crop[3]};
                INDArray out = cl.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
                assertArrayEquals(expShape, out.shape());

                String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = "
                        + Arrays.toString(crop);

                if (PRINT_RESULTS) {
                    System.out.println(msg);
                }

                boolean gradOK = GradientCheckUtil.checkGradients(
                        new GradientCheckUtil.MLNConfig().net(net)
                                .input(input).labels(labels)
                                .subset(true).maxPerParam(160));

                assertTrue(msg, gradOK);

                TestUtils.testModelSerialization(net);
            }
        }
    }
}
 

@Test
    public void testCnnZeroPaddingLayer() {
        Nd4j.getRandom().setSeed(12345);
        int nOut = 4;


        int width = 6;
        int height = 6;


        int[] kernel = {2, 2};
        int[] stride = {1, 1};
        int[] padding = {0, 0};

        int[] minibatchSizes = {1, 3, 2};
        int[] inputDepths = {1, 3, 2};
        int[][] zeroPadLayer = new int[][]{{0, 0, 0, 0}, {1, 1, 0, 0}, {2, 2, 2, 2}};

        for( int i=0; i<minibatchSizes.length; i++ ){
            int minibatchSize = minibatchSizes[i];
            int inputDepth = inputDepths[i];
            int[] zeroPad = zeroPadLayer[i];
            INDArray input = Nd4j.rand(DataType.DOUBLE, new int[]{minibatchSize, inputDepth, height, width});
            INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut);

            MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder().updater(new NoOp())
                            .dataType(DataType.DOUBLE)
                            .dist(new NormalDistribution(0, 1)).list()
                            .layer(0, new ConvolutionLayer.Builder(kernel, stride, padding)
                                    .nIn(inputDepth).nOut(3).build())//output: (6-2+0)/1+1 = 5
                            .layer(1, new ZeroPaddingLayer.Builder(zeroPad).build()).layer(2,
                            new ConvolutionLayer.Builder(kernel, stride,
                                    padding).nIn(3).nOut(3).build())//output: (6-2+0)/1+1 = 5
                            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(4).build())
                            .setInputType(InputType.convolutional(height, width, inputDepth))
                            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            //Check zero padding activation shape
            org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer zpl =
                    (org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer) net.getLayer(1);
            val expShape = new long[]{minibatchSize, inputDepth, height + zeroPad[0] + zeroPad[1],
                    width + zeroPad[2] + zeroPad[3]};
            INDArray out = zpl.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
            assertArrayEquals(expShape, out.shape());

            String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = "
                    + Arrays.toString(zeroPad);

            if (PRINT_RESULTS) {
                System.out.println(msg);
//                for (int j = 0; j < net.getnLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }
 

@Test
public void testLSTM() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    int minibatch = 4;
    int inputSize = 3;
    int lstmLayerSize = 4;
    int timeSeriesLength = 3;
    int nOut = 4;
    INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength});
    INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        for (int j = 0; j < timeSeriesLength; j++) {
            labels.putScalar(i, r.nextInt(nOut), j, 1.0);
        }
    }

    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp()).seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize)
                                    .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                    .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build())
                    ;

    MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
    mln.init();

    Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper");
    f.setAccessible(true);

    org.deeplearning4j.nn.layers.recurrent.LSTM l = (org.deeplearning4j.nn.layers.recurrent.LSTM) mln.getLayer(1);
    LSTMHelper helper = (LSTMHelper) f.get(l);
    assertTrue(helper instanceof CudnnLSTMHelper);

    //-------------------------------
    //For debugging/comparison to no-cudnn case: set helper field to null
    //        f.set(l, null);
    //        assertNull(f.get(l));
    //-------------------------------

    if (PRINT_RESULTS) {
        for (int j = 0; j < mln.getnLayers(); j++)
            System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
    }

    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, null, true, 32, null, null);

    assertTrue(gradOK);
}
 

@Test
public void testLRN() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int depth = 6;
    int hw = 5;
    int nOut = 4;
    INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw});
    INDArray labels = Nd4j.zeros(minibatch, nOut);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        labels.putScalar(i, r.nextInt(nOut), 1.0);
    }

    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new ConvolutionLayer.Builder().nOut(6).kernelSize(2, 2).stride(1, 1)
                                    .activation(Activation.TANH).build())
                    .layer(1, new LocalResponseNormalization.Builder().build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutional(hw, hw, depth));

    MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
    mln.init();

    Field f = org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization.class
                    .getDeclaredField("helper");
    f.setAccessible(true);

    org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization l =
                    (org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization) mln.getLayer(1);
    LocalResponseNormalizationHelper lrn = (LocalResponseNormalizationHelper) f.get(l);
    assertTrue(lrn instanceof CudnnLocalResponseNormalizationHelper);

    //-------------------------------
    //For debugging/comparison to no-cudnn case: set helper field to null
    //        f.set(l, null);
    //        assertNull(f.get(l));
    //-------------------------------

    if (PRINT_RESULTS) {
        for (int j = 0; j < mln.getnLayers(); j++)
            System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
    }

    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

    assertTrue(gradOK);
}
 

@Test
public void testBatchNormCnn() throws Exception {
    //Note: CuDNN batch norm supports 4d only, as per 5.1 (according to api reference documentation)
    Nd4j.getRandom().setSeed(12345);
    int minibatch = 10;
    int depth = 1;
    int hw = 4;
    int nOut = 4;
    INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw});
    INDArray labels = Nd4j.zeros(minibatch, nOut);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        labels.putScalar(i, r.nextInt(nOut), 1.0);
    }

    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 2)).list()
                    .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2)
                                    .activation(Activation.IDENTITY).build())
                    .layer(1, new BatchNormalization.Builder().build())
                    .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutional(hw, hw, depth));

    MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
    mln.init();

    Field f = org.deeplearning4j.nn.layers.normalization.BatchNormalization.class.getDeclaredField("helper");
    f.setAccessible(true);

    org.deeplearning4j.nn.layers.normalization.BatchNormalization b =
                    (org.deeplearning4j.nn.layers.normalization.BatchNormalization) mln.getLayer(1);
    BatchNormalizationHelper bn = (BatchNormalizationHelper) f.get(b);
    assertTrue(bn instanceof CudnnBatchNormalizationHelper);

    //-------------------------------
    //For debugging/comparison to no-cudnn case: set helper field to null
    //        f.set(b, null);
    //        assertNull(f.get(b));
    //-------------------------------

    if (PRINT_RESULTS) {
        for (int j = 0; j < mln.getnLayers(); j++)
            System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
    }

    //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
    //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
    //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
    Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, null, false, -1, excludeParams, null);

    assertTrue(gradOK);
}
 

@Test
public void testConvolutional() throws Exception {

    //Parameterized test, testing combinations of:
    // (a) activation function
    // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
    // (c) Loss function (with specified output activations)
    Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
    boolean[] characteristic = {false, true}; //If true: run some backprop steps first

    int[] minibatchSizes = {1, 4};
    int width = 6;
    int height = 6;
    int inputDepth = 2;
    int nOut = 3;

    Field f = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper");
    f.setAccessible(true);

    Random r = new Random(12345);
    for (Activation afn : activFns) {
        for (boolean doLearningFirst : characteristic) {
            for (int minibatchSize : minibatchSizes) {

                INDArray input = Nd4j.rand(new int[] {minibatchSize, inputDepth, height, width});
                INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                for (int i = 0; i < minibatchSize; i++) {
                    labels.putScalar(i, r.nextInt(nOut), 1.0);
                }

                MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                        .dataType(DataType.DOUBLE)
                                .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT)
                                .dist(new UniformDistribution(-1, 1))
                                .updater(new NoOp()).seed(12345L).list()
                                .layer(0, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(1, 1).nOut(3)
                                                .activation(afn).build())
                                .layer(1, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(0, 0).nOut(3)
                                                .activation(afn).build())
                                .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                .activation(Activation.SOFTMAX).nOut(nOut).build())
                                .setInputType(InputType.convolutional(height, width, inputDepth))
                                ;

                MultiLayerConfiguration conf = builder.build();

                MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                mln.init();

                org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c0 =
                                (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(0);
                ConvolutionHelper ch0 = (ConvolutionHelper) f.get(c0);
                assertTrue(ch0 instanceof CudnnConvolutionHelper);

                org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c1 =
                                (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(1);
                ConvolutionHelper ch1 = (ConvolutionHelper) f.get(c1);
                assertTrue(ch1 instanceof CudnnConvolutionHelper);

                //-------------------------------
                //For debugging/comparison to no-cudnn case: set helper field to null
                //                    f.set(c0, null);
                //                    f.set(c1, null);
                //                    assertNull(f.get(c0));
                //                    assertNull(f.get(c1));
                //-------------------------------


                String name = new Object() {}.getClass().getEnclosingMethod().getName();

                if (doLearningFirst) {
                    //Run a number of iterations of learning
                    mln.setInput(input);
                    mln.setLabels(labels);
                    mln.computeGradientAndScore();
                    double scoreBefore = mln.score();
                    for (int j = 0; j < 10; j++)
                        mln.fit(input, labels);
                    mln.computeGradientAndScore();
                    double scoreAfter = mln.score();
                    //Can't test in 'characteristic mode of operation' if not learning
                    String msg = name + " - score did not (sufficiently) decrease during learning - activationFn="
                                    + afn + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                                    + ", scoreAfter=" + scoreAfter + ")";
                    assertTrue(msg, scoreAfter < 0.8 * scoreBefore);
                }

                if (PRINT_RESULTS) {
                    System.out.println(name + " - activationFn=" + afn + ", doLearningFirst=" + doLearningFirst);
                    for (int j = 0; j < mln.getnLayers(); j++)
                        System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                }

                boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                assertTrue(gradOK);
            }
        }
    }
}
 

@Test
    public void testYoloActivateScoreBasic() {

        //Note that we expect some NaNs here - 0/0 for example in IOU calculation. This is handled explicitly in the
        //implementation
        //Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.ANY_PANIC);

        int mb = 3;
        int b = 4;
        int c = 3;
        int depth = b * (5 + c);
        int w = 6;
        int h = 6;

        INDArray bbPrior = Nd4j.rand(b, 2).muliRowVector(Nd4j.create(new double[]{w, h}));


        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .l2(0.01)
                .list()
                .layer(new ConvolutionLayer.Builder().nIn(depth).nOut(depth).kernelSize(1,1).build())
                .layer(new Yolo2OutputLayer.Builder()
                        .boundingBoxPriors(bbPrior)
                        .build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer y2impl = (org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer) net.getLayer(1);

        INDArray input = Nd4j.rand(new int[]{mb, depth, h, w});

        INDArray out = y2impl.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
        assertNotNull(out);
        assertArrayEquals(input.shape(), out.shape());



        //Check score method (simple)
        int labelDepth = 4 + c;
        INDArray labels = Nd4j.zeros(mb, labelDepth, h, w);
        //put 1 object per minibatch, at positions (0,0), (1,1) etc.
        //Positions for label boxes: (1,1) to (2,2), (2,2) to (4,4) etc
        labels.putScalar(0, 4 + 0, 0, 0, 1);
        labels.putScalar(1, 4 + 1, 1, 1, 1);
        labels.putScalar(2, 4 + 2, 2, 2, 1);

        labels.putScalar(0, 0, 0, 0, 1);
        labels.putScalar(0, 1, 0, 0, 1);
        labels.putScalar(0, 2, 0, 0, 2);
        labels.putScalar(0, 3, 0, 0, 2);

        labels.putScalar(1, 0, 1, 1, 2);
        labels.putScalar(1, 1, 1, 1, 2);
        labels.putScalar(1, 2, 1, 1, 4);
        labels.putScalar(1, 3, 1, 1, 4);

        labels.putScalar(2, 0, 2, 2, 3);
        labels.putScalar(2, 1, 2, 2, 3);
        labels.putScalar(2, 2, 2, 2, 6);
        labels.putScalar(2, 3, 2, 2, 6);

        y2impl.setInput(input, LayerWorkspaceMgr.noWorkspaces());
        y2impl.setLabels(labels);
        double score = y2impl.computeScore(0.0, true, LayerWorkspaceMgr.noWorkspaces());

//        System.out.println("SCORE: " + score);
        assertTrue(score > 0.0);


        //Finally: test ser/de:
        MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net);

        y2impl = (org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer) netLoaded.getLayer(1);
        y2impl.setInput(input, LayerWorkspaceMgr.noWorkspaces());
        y2impl.setLabels(labels);
        double score2 = y2impl.computeScore(0.0, true, LayerWorkspaceMgr.noWorkspaces());

        assertEquals(score, score2, 1e-8);

        //Test computeScoreForExamples:
        INDArray scoreArr1 = net.scoreExamples(new DataSet(input, labels), false);
        INDArray scoreArr2 = net.scoreExamples(new DataSet(input, labels), true);
        assertFalse(scoreArr1.isAttached());
        assertFalse(scoreArr2.isAttached());

        assertArrayEquals(new long[]{mb,1}, scoreArr1.shape());
        assertArrayEquals(new long[]{mb,1}, scoreArr2.shape());
        assertNotEquals(scoreArr1, scoreArr2);
    }
 

@Test
public void validateImplMultiLayerRnnTimeStep() throws Exception {

    for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) {
        Nd4j.getRandom().setSeed(12345);
        int minibatch = 10;
        int inputSize = 3;
        int lstmLayerSize = 4;
        int timeSeriesLength = 3;
        int tbpttLength = 5;
        int nOut = 2;

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp())
                .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE)
                .cacheMode(CacheMode.NONE).seed(12345L)
                .dist(new NormalDistribution(0, 2)).list()
                .layer(0, new LSTM.Builder().nIn(inputSize).nOut(lstmLayerSize)
                        .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize)
                        .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build())
                .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                        .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build())
                .backpropType(BackpropType.TruncatedBPTT)
                .tBPTTLength(tbpttLength).build();

        MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone());
        mln1.init();

        MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone());
        mln2.init();


        assertEquals(mln1.params(), mln2.params());

        Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper");
        f.setAccessible(true);

        Layer l0 = mln1.getLayer(0);
        Layer l1 = mln1.getLayer(1);
        f.set(l0, null);
        f.set(l1, null);
        assertNull(f.get(l0));
        assertNull(f.get(l1));

        l0 = mln2.getLayer(0);
        l1 = mln2.getLayer(1);
        assertTrue(f.get(l0) instanceof CudnnLSTMHelper);
        assertTrue(f.get(l1) instanceof CudnnLSTMHelper);

        Random r = new Random(12345);
        for (int x = 0; x < 5; x++) {
            INDArray input = Nd4j.rand(new int[]{minibatch, inputSize, timeSeriesLength});

            INDArray step1 = mln1.rnnTimeStep(input);
            INDArray step2 = mln2.rnnTimeStep(input);

            assertEquals("Step: " + x, step1, step2);
        }

        assertEquals(mln1.params(), mln2.params());

        //Also check fit (mainly for workspaces sanity check):
        INDArray in = Nd4j.rand(new int[]{minibatch, inputSize, 3 * tbpttLength});
        INDArray label = TestUtils.randomOneHotTimeSeries(minibatch, nOut, 3 * tbpttLength);
        for( int i=0; i<3; i++ ){
            mln1.fit(in, label);
            mln2.fit(in, label);
        }
    }
}
 

@Test
public void testCropping2DLayer() {
    Nd4j.getRandom().setSeed(12345);
    int nOut = 2;
    int width = 12;
    int height = 11;

    int[] kernel = {2, 2};
    int[] stride = {1, 1};
    int[] padding = {0, 0};

    int[][] cropTestCases = new int[][]{{0, 0, 0, 0}, {1, 1, 0, 0}, {2, 2, 2, 2}, {1, 2, 3, 4}};
    int[] inputDepths = {1, 2, 3, 2};
    int[] minibatchSizes = {2, 1, 3, 2};

    boolean nchw = format == CNN2DFormat.NCHW;

    for (int i = 0; i < cropTestCases.length; i++) {
        int inputDepth = inputDepths[i];
        int minibatchSize = minibatchSizes[i];
        int[] crop = cropTestCases[i];
        long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth};
        INDArray input = Nd4j.rand(DataType.DOUBLE, inShape);
        INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut);

        MultiLayerConfiguration conf =
                new NeuralNetConfiguration.Builder()
                        .dataType(DataType.DOUBLE)
                        .updater(new NoOp())
                        .convolutionMode(ConvolutionMode.Same)
                        .weightInit(new NormalDistribution(0, 1)).list()
                        .layer(new ConvolutionLayer.Builder(kernel, stride, padding)
                                .nIn(inputDepth).nOut(2).build())//output: (6-2+0)/1+1 = 5
                        .layer(new Cropping2D(crop))
                        .layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(2).nOut(2).build())
                        .layer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG).kernelSize(3, 3).stride(3, 3).build())
                        .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                .activation(Activation.SOFTMAX).nOut(nOut).build())
                        .setInputType(InputType.convolutional(height, width, inputDepth, format))
                        .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        //Check cropping activation shape
        org.deeplearning4j.nn.layers.convolution.Cropping2DLayer cl =
                (org.deeplearning4j.nn.layers.convolution.Cropping2DLayer) net.getLayer(1);
        long[] expShape;
        if(nchw){
            expShape = new long[]{minibatchSize, inputDepth, height - crop[0] - crop[1],
                    width - crop[2] - crop[3]};
        } else {
            expShape = new long[]{minibatchSize, height - crop[0] - crop[1],
                    width - crop[2] - crop[3], inputDepth};
        }
        INDArray out = cl.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
        assertArrayEquals(expShape, out.shape());

        String msg = format + " - minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = "
                + Arrays.toString(crop);

        if (PRINT_RESULTS) {
            System.out.println(msg);
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input)
                .labels(labels).subset(true).maxPerParam(160));

        assertTrue(msg, gradOK);

        TestUtils.testModelSerialization(net);
    }
}
 

@Test
    public void testCnnZeroPaddingLayer() {
        Nd4j.getRandom().setSeed(12345);
        int nOut = 4;


        int width = 6;
        int height = 6;


        int[] kernel = {2, 2};
        int[] stride = {1, 1};
        int[] padding = {0, 0};

        int[] minibatchSizes = {1, 3, 2};
        int[] inputDepths = {1, 3, 2};
        int[][] zeroPadLayer = new int[][]{{0, 0, 0, 0}, {1, 1, 0, 0}, {2, 2, 2, 2}};

        boolean nchw = format == CNN2DFormat.NCHW;

        for( int i=0; i<minibatchSizes.length; i++ ){
            int minibatchSize = minibatchSizes[i];
            int inputDepth = inputDepths[i];
            int[] zeroPad = zeroPadLayer[i];

            long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth};
            INDArray input = Nd4j.rand(DataType.DOUBLE, inShape);
            INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut);

            MultiLayerConfiguration conf =
                    new NeuralNetConfiguration.Builder().updater(new NoOp())
                            .dataType(DataType.DOUBLE)
                            .dist(new NormalDistribution(0, 1)).list()
                            .layer(0, new ConvolutionLayer.Builder(kernel, stride, padding)
                                    .nIn(inputDepth).nOut(3).build())//output: (6-2+0)/1+1 = 5
                            .layer(1, new ZeroPaddingLayer.Builder(zeroPad).build()).layer(2,
                            new ConvolutionLayer.Builder(kernel, stride,
                                    padding).nIn(3).nOut(3).build())//output: (6-2+0)/1+1 = 5
                            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(4).build())
                            .setInputType(InputType.convolutional(height, width, inputDepth, format))
                            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            //Check zero padding activation shape
            org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer zpl =
                    (org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer) net.getLayer(1);
            long[] expShape;
            if(nchw){
                expShape = new long[]{minibatchSize, inputDepth, height + zeroPad[0] + zeroPad[1],
                        width + zeroPad[2] + zeroPad[3]};
            } else {
                expShape = new long[]{minibatchSize, height + zeroPad[0] + zeroPad[1],
                        width + zeroPad[2] + zeroPad[3], inputDepth};
            }
            INDArray out = zpl.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
            assertArrayEquals(expShape, out.shape());

            String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = "
                    + Arrays.toString(zeroPad);

            if (PRINT_RESULTS) {
                System.out.println(msg);
//                for (int j = 0; j < net.getnLayers(); j++)
//                    System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
            }

            boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                    DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }