Java Code Examples for org.deeplearning4j.nn.api.Layer#backpropGradient()

The following examples show how to use org.deeplearning4j.nn.api.Layer#backpropGradient() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SubsamplingLayerTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSubSampleLayerAvgBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 2., 3., 4., 5., 6., 7., 8.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5,
                    0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75,
                    2., 2., 1.75, 1.75, 2., 2.}, new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());
    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape());

}
Example 2
Source File: RepeatVectorTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRepeatVector() {

    double[] arr = new double[] {1., 2., 3., 1., 2., 3., 1., 2., 3., 1., 2., 3.};
    INDArray expectedOut = Nd4j.create(arr, new long[] {1, 3, REPEAT}, 'f');
    INDArray input = Nd4j.create(new double[] {1., 2., 3.}, new long[] {1, 3});
    Layer layer = getRepeatVectorLayer();

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(expectedOut.shape(), output.shape()));
    assertEquals(expectedOut, output);

    INDArray epsilon = Nd4j.ones(1,3,4);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    INDArray outEpsilon = out.getSecond();
    INDArray expectedEpsilon = Nd4j.create(new double[] {4., 4., 4.}, new long[] {1, 3});
    assertEquals(expectedEpsilon, outEpsilon);
}
Example 3
Source File: Upsampling1DTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUpsampling1DBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 3., 2., 6., 7., 2., 5., 5.},
                            new int[] {1, 1, 8});

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 8., 9., 10.},
                    new int[] {1, 1, 4});

    INDArray input = getContainedData();

    Layer layer = getUpsampling1DLayer();
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    val depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, outputLength);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1));
}
Example 4
Source File: Upsampling2DTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUpsampling2DBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.},
                            new int[] {1, 1, 4, 4});

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 4., 4., 4.},
                    new int[] {1, 1, 2, 2});

    INDArray input = getContainedData();

    Layer layer = getUpsamplingLayer();
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    val depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, outputHeight, outputWidth);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1));
}
Example 5
Source File: SubsamplingLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSubSampleLayerMaxBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1.,
                    0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.},
                    new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());

    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    long depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1)); // channels retained
}
Example 6
Source File: SubsamplingLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testSubSampleLayerSumBackprop() throws Exception {
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.SUM);
    INDArray input = getData();
    layer.setInput(input, LayerWorkspaceMgr.noWorkspaces());
    layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
}
Example 7
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
    public void testDnnForwardBackward() {
        double eps = 1e-5;
        int nIn = 4;
        int minibatch = 2;
        Nd4j.getRandom().setSeed(12345);
        INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn});

        //TODO: other values for gamma/beta
        INDArray gamma = Nd4j.ones(1, nIn);
        INDArray beta = Nd4j.zeros(1, nIn);

        Layer l = getLayer(nIn, eps, false, -1, -1);

        INDArray mean = input.mean(0);
        INDArray var = input.var(false, 0);
        INDArray xHat = input.subRowVector(mean).divRowVector(Transforms.sqrt(var.add(eps), true));
        INDArray outExpected = xHat.mulRowVector(gamma).addRowVector(beta);

        INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces());

//        System.out.println(Arrays.toString(outExpected.data().asDouble()));
//        System.out.println(Arrays.toString(out.data().asDouble()));

        assertEquals(outExpected, out);

        //-------------------------------------------------------------
        //Check backprop
        INDArray epsilon = Nd4j.rand(minibatch, nIn); //dL/dy

        INDArray dldgammaExp = epsilon.mul(xHat).sum(true, 0);
        INDArray dldbetaExp = epsilon.sum(true, 0);

        INDArray dldxhat = epsilon.mulRowVector(gamma);
        INDArray dldvar = dldxhat.mul(input.subRowVector(mean)).mul(-0.5)
                .mulRowVector(Transforms.pow(var.add(eps), -3.0 / 2.0, true)).sum(0);
        INDArray dldmu = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)).neg().sum(0)
                .add(dldvar.mul(input.subRowVector(mean).mul(-2.0).sum(0).div(minibatch)));
        INDArray dldinExp = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true))
                .add(input.subRowVector(mean).mul(2.0 / minibatch).mulRowVector(dldvar))
                .addRowVector(dldmu.mul(1.0 / minibatch));

        Pair<Gradient, INDArray> p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());

        INDArray dldgamma = p.getFirst().getGradientFor("gamma");
        INDArray dldbeta = p.getFirst().getGradientFor("beta");

        assertEquals(dldgammaExp, dldgamma);
        assertEquals(dldbetaExp, dldbeta);

//        System.out.println("EPSILONS");
//        System.out.println(Arrays.toString(dldinExp.data().asDouble()));
//        System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
        assertEquals(dldinExp, p.getSecond());
    }
Example 8
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void test2dVs4d() {
    //Idea: 2d and 4d should be the same...
    Nd4j.getRandom().setSeed(12345);

    int m = 2;
    int h = 3;
    int w = 3;
    int nOut = 2;

    INDArray in = Nd4j.rand('c', m * h * w, nOut);

    INDArray in4 = in.dup();
    in4 = Shape.newShapeNoCopy(in4, new int[]{m, h, w, nOut}, false);
    assertNotNull(in4);
    in4 = in4.permute(0, 3, 1, 2).dup();
    INDArray arr = Nd4j.rand(1, m * h * w * nOut).reshape('f', h, w, m, nOut).permute(2, 3, 1, 0);
    in4 = arr.assign(in4);

    Layer l1 = getLayer(nOut);
    Layer l2 = getLayer(nOut);

    INDArray out2d = l1.activate(in.dup(), true, LayerWorkspaceMgr.noWorkspaces());
    INDArray out4d = l2.activate(in4.dup(), true, LayerWorkspaceMgr.noWorkspaces());

    INDArray out4dAs2 = out4d.permute(0, 2, 3, 1).dup('c');
    out4dAs2 = Shape.newShapeNoCopy(out4dAs2, new int[]{m * h * w, nOut}, false);

    assertEquals(out2d, out4dAs2);

    //Test backprop:
    INDArray epsilons2d = Nd4j.rand('c', m * h * w, nOut);
    INDArray epsilons4d = epsilons2d.dup();
    epsilons4d = Shape.newShapeNoCopy(epsilons4d, new int[]{m, h, w, nOut}, false);
    assertNotNull(epsilons4d);
    epsilons4d = epsilons4d.permute(0, 3, 1, 2).dup();

    Pair<Gradient, INDArray> b2d = l1.backpropGradient(epsilons2d, LayerWorkspaceMgr.noWorkspaces());
    Pair<Gradient, INDArray> b4d = l2.backpropGradient(epsilons4d, LayerWorkspaceMgr.noWorkspaces());

    INDArray e4dAs2d = b4d.getSecond().permute(0, 2, 3, 1).dup('c');
    e4dAs2d = Shape.newShapeNoCopy(e4dAs2d, new int[]{m * h * w, nOut}, false);

    assertEquals(b2d.getSecond(), e4dAs2d);
}
Example 9
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
    public void testCnnForwardBackward() {
        double eps = 1e-5;
        int nIn = 4;
        int hw = 3;
        int minibatch = 2;
        Nd4j.getRandom().setSeed(12345);
        INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw});

        //TODO: other values for gamma/beta
        INDArray gamma = Nd4j.ones(1, nIn);
        INDArray beta = Nd4j.zeros(1, nIn);

        Layer l = getLayer(nIn, eps, false, -1, -1);

        INDArray mean = input.mean(0, 2, 3);
        INDArray var = input.var(false, 0, 2, 3);
        INDArray xHat = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));
        Nd4j.getExecutioner().exec(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1));

        INDArray outExpected = Nd4j.getExecutioner().exec(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1));
        Nd4j.getExecutioner().exec(new BroadcastAddOp(outExpected, beta, outExpected, 1));

        INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces());

//        System.out.println(Arrays.toString(outExpected.data().asDouble()));
//        System.out.println(Arrays.toString(out.data().asDouble()));

        assertEquals(outExpected, out);

        //-------------------------------------------------------------
        //Check backprop
        INDArray epsilon = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw}); //dL/dy

        int effectiveMinibatch = minibatch * hw * hw;

        INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3);
        dldgammaExp = dldgammaExp.reshape(1, dldgammaExp.length());
        INDArray dldbetaExp = epsilon.sum(0, 2, 3);
        dldbetaExp = dldbetaExp.reshape(1, dldbetaExp.length());

        INDArray dldxhat = Nd4j.getExecutioner().exec(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1)); //epsilon.mulRowVector(gamma);

        INDArray inputSubMean = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));

        INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5);
        dldvar = Nd4j.getExecutioner().exec(
                new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1));
        dldvar = dldvar.sum(0, 2, 3);


        INDArray dldmu = Nd4j
                .getExecutioner().exec(new BroadcastMulOp(dldxhat,
                        Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1))
                .neg().sum(0, 2, 3);
        dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch)));

        INDArray dldinExp = Nd4j.getExecutioner().exec(
                new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1));
        dldinExp = dldinExp.add(Nd4j.getExecutioner().exec(
                new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1)));
        dldinExp = Nd4j.getExecutioner().exec(
                new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1));

        Pair<Gradient, INDArray> p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());

        INDArray dldgamma = p.getFirst().getGradientFor("gamma");
        INDArray dldbeta = p.getFirst().getGradientFor("beta");

        assertEquals(dldgammaExp, dldgamma);
        assertEquals(dldbetaExp, dldbeta);

        //        System.out.println("EPSILONS");
        //        System.out.println(Arrays.toString(dldinExp.data().asDouble()));
        //        System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
        assertEquals(dldinExp, p.getSecond());
    }