Java Code Examples for org.nd4j.linalg.ops.transforms.Transforms#sigmoid()

@Test
public void testTranspose() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 4, 4));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.transpose(x);
    sameDiff.exec();
    assertArrayEquals(new long[]{4, 1}, result.getArr().shape());

}
 

@Test
public void testSigmoid() {
    INDArray n = Nd4j.create(new float[] {1, 2, 3, 4});
    INDArray assertion = Nd4j.create(new float[] {0.73105858f, 0.88079708f, 0.95257413f, 0.98201379f});
    INDArray sigmoid = Transforms.sigmoid(n, false);
    assertEquals(getFailureMessage(), assertion, sigmoid);

}
 

@Test
public void testTile2() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 4, 4, DataType.DOUBLE).reshape(1,4));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.tile(x, new int[]{2, 2});
    assertArrayEquals(new long[]{2, 8}, result.eval().shape());
    INDArray arr2 = Nd4j.concat(0, arr, arr);  // (1, 4), (1, 4) -> (2, 4)
    INDArray expected = Nd4j.concat(1, arr2, arr2);  // (2, 4), (2, 4) -> (2, 8)
    assertEquals(expected, result.eval());
}
 

@Test
public void testPermuteSimple() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 6, 6).reshape(2, 3));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.permute(x, 1, 0);
    Map<String,INDArray> m = sameDiff.outputAll(null);
    assertArrayEquals(new long[]{3, 2}, m.get(result.name()).shape());

}
 

@Test
public void testDenseLayerForwardPass() {
    Nd4j.getRandom().setSeed(12345);

    SameDiff sd = SameDiff.create();

    INDArray iInput = Nd4j.rand(3, 4);
    INDArray iWeights = Nd4j.rand(4, 5);
    INDArray iBias = Nd4j.rand(1, 5);

    SDVariable input = sd.var("input", iInput);
    SDVariable weights = sd.var("weights", iWeights);
    SDVariable bias = sd.var("bias", iBias);

    SDVariable mmul = sd.mmul("mmul", input, weights);
    SDVariable z = mmul.add("z", bias);
    SDVariable out = sd.nn().sigmoid("out", z);

    INDArray expMmul = iInput.mmul(iWeights);
    INDArray expZ = expMmul.addRowVector(iBias);
    INDArray expOut = Transforms.sigmoid(expZ, true);

    Map<String,INDArray> m = sd.outputAll(Collections.emptyMap());

    assertEquals(expMmul, m.get(mmul.name()));
    assertEquals(expZ, m.get(z.name()));
    assertEquals(expOut, m.get(out.name()));
}
 

@Test
public void testEval() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Nd4j.linspace(1, 4, 4);
    SDVariable x = sameDiff.var("x", arr);
    SDVariable sigmoid = sameDiff.nn().sigmoid("s", x);
    INDArray assertion = Transforms.sigmoid(arr);
    INDArray eval = sameDiff.output(Collections.singletonMap("x", arr), Collections.singletonList("s")).get("s");
    assertEquals(assertion, eval);
}
 

@Test
public void testSigmoid() {
    INDArray n = Nd4j.create(new float[] {1, 2, 3, 4});
    INDArray assertion = Nd4j.create(new float[] {0.73105858f, 0.88079708f, 0.95257413f, 0.98201379f});
    INDArray sigmoid = Transforms.sigmoid(n, false);
    assertEquals(getFailureMessage(), assertion, sigmoid);

}
 

@Test
public void testDenseLayerForwardPass() {
    Nd4j.getRandom().setSeed(12345);

    SameDiff sd = SameDiff.create();

    INDArray iInput = Nd4j.rand(3, 4);
    INDArray iWeights = Nd4j.rand(4, 5);
    INDArray iBias = Nd4j.rand(1, 5);

    SDVariable input = sd.var("input", iInput);
    SDVariable weights = sd.var("weights", iWeights);
    SDVariable bias = sd.var("bias", iBias);

    SDVariable mmul = sd.mmul("mmul", input, weights);
    SDVariable z = mmul.add("z", bias);
    SDVariable out = sd.sigmoid("out", z);

    INDArray expMmul = iInput.mmul(iWeights);
    INDArray expZ = expMmul.addRowVector(iBias);
    INDArray expOut = Transforms.sigmoid(expZ, true);

    sd.exec();

    assertEquals(expMmul, mmul.getArr());
    assertEquals(expZ, z.getArr());
    assertEquals(expOut, out.getArr());
}
 

@Test
public void testEval() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Nd4j.linspace(1, 4, 4);
    SDVariable x = sameDiff.var("x", arr);
    SDVariable sigmoid = sameDiff.sigmoid(x);
    INDArray assertion = Transforms.sigmoid(arr);
    INDArray[] eval = sameDiff.eval(Collections.singletonMap("x", arr));
    assertEquals(assertion, eval[0]);

}
 

@Test
public void testTile() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 4, 4));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.tile(x, new int[]{2, 2});
    assertArrayEquals(new long[]{2, 8}, result.eval().shape());
    INDArray arr2 = Nd4j.concat(0, arr, arr);  // (1, 4), (1, 4) -> (2, 4)
    INDArray expected = Nd4j.concat(1, arr2, arr2);  // (2, 4), (2, 4) -> (2, 8)
    assertEquals(expected, result.eval());

}
 

@Test
public void testConcat() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr1 = Transforms.sigmoid(Nd4j.linspace(1, 4, 4));
    INDArray arr2 = Transforms.sigmoid(Nd4j.linspace(4, 8, 4));
    SDVariable x1 = sameDiff.var("x1", arr1);
    SDVariable x2 = sameDiff.var("x2", arr2);
    SDVariable result = sameDiff.concat(0, x1, x2);
    assertArrayEquals(new long[]{2, 4}, result.eval().shape());

}
 

@Test
public void testPermute() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 6, 6).reshape(2, 3));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.permute(x, 1, 0);
    assertArrayEquals(new long[]{3, 2}, result.getShape());

}
 

@Test
public void testSum() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Transforms.sigmoid(Nd4j.linspace(1, 4, 4));
    SDVariable x = sameDiff.var("x", arr);
    SDVariable result = sameDiff.sum(x, 1); //[1,4].sum(1) == [1,1]
    assertArrayEquals(new long[]{1, 1}, result.getShape());
}
 

@Test
    public void testActivationBackprop() {

        Activation[] afns = new Activation[]{
                Activation.TANH,
                Activation.SIGMOID,
                Activation.ELU,
                Activation.SOFTPLUS,
                Activation.SOFTSIGN,
                Activation.HARDTANH,
                Activation.CUBE,            //WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
                Activation.RELU,            //JVM crash
                Activation.LEAKYRELU        //JVM crash
        };

        for (Activation a : afns) {

            SameDiff sd = SameDiff.create();
            INDArray inArr = Nd4j.linspace(-3, 3, 7);
            INDArray labelArr = Nd4j.linspace(-3, 3, 7).muli(0.5);
            SDVariable in = sd.var("in", inArr.dup());

//            System.out.println("inArr: " + inArr);

            INDArray outExp;
            SDVariable out;
            switch (a) {
                case ELU:
                    out = sd.elu("out", in);
                    outExp = Transforms.elu(inArr, true);
                    break;
                case HARDTANH:
                    out = sd.hardTanh("out", in);
                    outExp = Transforms.hardTanh(inArr, true);
                    break;
                case LEAKYRELU:
                    out = sd.leakyRelu("out", in, 0.01);
                    outExp = Transforms.leakyRelu(inArr, true);
                    break;
                case RELU:
                    out = sd.relu("out", in, 0.0);
                    outExp = Transforms.relu(inArr, true);
                    break;
                case SIGMOID:
                    out = sd.sigmoid("out", in);
                    outExp = Transforms.sigmoid(inArr, true);
                    break;
                case SOFTPLUS:
                    out = sd.softplus("out", in);
                    outExp = Transforms.softPlus(inArr, true);
                    break;
                case SOFTSIGN:
                    out = sd.softsign("out", in);
                    outExp = Transforms.softsign(inArr, true);
                    break;
                case TANH:
                    out = sd.tanh("out", in);
                    outExp = Transforms.tanh(inArr, true);
                    break;
                case CUBE:
                    out = sd.cube("out", in);
                    outExp = Transforms.pow(inArr, 3, true);
                    break;
                default:
                    throw new RuntimeException(a.toString());
            }

            //Sum squared error loss:
            SDVariable label = sd.var("label", labelArr.dup());
            SDVariable diff = label.sub("diff", out);
            SDVariable sqDiff = diff.mul("sqDiff", diff);
            SDVariable totSum = sd.sum("totSum", sqDiff, Integer.MAX_VALUE);    //Loss function...

            sd.exec();
            INDArray outAct = sd.getVariable("out").getArr();
            assertEquals(a.toString(), outExp, outAct);

            // L = sum_i (label - out)^2
            //dL/dOut = 2(out - label)
            INDArray dLdOutExp = outExp.sub(labelArr).mul(2);
            INDArray dLdInExp = a.getActivationFunction().backprop(inArr.dup(), dLdOutExp.dup()).getFirst();

            sd.execBackwards();
            SameDiff gradFn = sd.getFunction("grad");
            INDArray dLdOutAct = gradFn.getVariable("out-grad").getArr();
            INDArray dLdInAct = gradFn.getVariable("in-grad").getArr();

            assertEquals(a.toString(), dLdOutExp, dLdOutAct);
            assertEquals(a.toString(), dLdInExp, dLdInAct);
        }
    }
 

@Test
    public void testActivationBackprop() {

        Activation[] afns = new Activation[]{
                Activation.TANH,
                Activation.SIGMOID,
                Activation.ELU,
                Activation.SOFTPLUS,
                Activation.SOFTSIGN,
                Activation.HARDTANH,
                Activation.CUBE,
                //WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
                Activation.RELU,            //JVM crash
                Activation.LEAKYRELU        //JVM crash
        };

        for (Activation a : afns) {

            SameDiff sd = SameDiff.create();
            INDArray inArr = Nd4j.linspace(-3, 3, 7);
            INDArray labelArr = Nd4j.linspace(-3, 3, 7).muli(0.5);
            SDVariable in = sd.var("in", inArr.dup());

//            System.out.println("inArr: " + inArr);

            INDArray outExp;
            SDVariable out;
            switch (a) {
                case ELU:
                    out = sd.nn().elu("out", in);
                    outExp = Transforms.elu(inArr, true);
                    break;
                case HARDTANH:
                    out = sd.nn().hardTanh("out", in);
                    outExp = Transforms.hardTanh(inArr, true);
                    break;
                case LEAKYRELU:
                    out = sd.nn().leakyRelu("out", in, 0.01);
                    outExp = Transforms.leakyRelu(inArr, true);
                    break;
                case RELU:
                    out = sd.nn().relu("out", in, 0.0);
                    outExp = Transforms.relu(inArr, true);
                    break;
                case SIGMOID:
                    out = sd.nn().sigmoid("out", in);
                    outExp = Transforms.sigmoid(inArr, true);
                    break;
                case SOFTPLUS:
                    out = sd.nn().softplus("out", in);
                    outExp = Transforms.softPlus(inArr, true);
                    break;
                case SOFTSIGN:
                    out = sd.nn().softsign("out", in);
                    outExp = Transforms.softsign(inArr, true);
                    break;
                case TANH:
                    out = sd.math().tanh("out", in);
                    outExp = Transforms.tanh(inArr, true);
                    break;
                case CUBE:
                    out = sd.math().cube("out", in);
                    outExp = Transforms.pow(inArr, 3, true);
                    break;
                default:
                    throw new RuntimeException(a.toString());
            }

            //Sum squared error loss:
            SDVariable label = sd.var("label", labelArr.dup());
            SDVariable diff = label.sub("diff", out);
            SDVariable sqDiff = diff.mul("sqDiff", diff);
            SDVariable totSum = sd.sum("totSum", sqDiff, Integer.MAX_VALUE);    //Loss function...

            Map<String,INDArray> m = sd.output(Collections.emptyMap(), "out");
            INDArray outAct = m.get("out");
            assertEquals(a.toString(), outExp, outAct);

            // L = sum_i (label - out)^2
            //dL/dOut = 2(out - label)
            INDArray dLdOutExp = outExp.sub(labelArr).mul(2);
            INDArray dLdInExp = a.getActivationFunction().backprop(inArr.dup(), dLdOutExp.dup()).getFirst();

            Map<String,INDArray> grads = sd.calculateGradients(null, "out", "in");
//            sd.execBackwards(Collections.emptyMap());
//            SameDiff gradFn = sd.getFunction("grad");
            INDArray dLdOutAct = grads.get("out");
            INDArray dLdInAct = grads.get("in");

            assertEquals(a.toString(), dLdOutExp, dLdOutAct);
            assertEquals(a.toString(), dLdInExp, dLdInAct);
        }
    }
 

@Test
public void testRnnBlockCell(){
    Nd4j.getRandom().setSeed(12345);
    int mb = 2;
    int nIn = 3;
    int nOut = 4;

    SameDiff sd = SameDiff.create();
    SDVariable x = sd.constant(Nd4j.rand(DataType.FLOAT, mb, nIn));
    SDVariable cLast = sd.constant(Nd4j.rand(DataType.FLOAT, mb, nOut));
    SDVariable yLast = sd.constant(Nd4j.rand(DataType.FLOAT, mb, nOut));
    SDVariable W = sd.constant(Nd4j.rand(DataType.FLOAT, (nIn+nOut), 4*nOut));
    SDVariable Wci = sd.constant(Nd4j.rand(DataType.FLOAT, nOut));
    SDVariable Wcf = sd.constant(Nd4j.rand(DataType.FLOAT, nOut));
    SDVariable Wco = sd.constant(Nd4j.rand(DataType.FLOAT, nOut));
    SDVariable b = sd.constant(Nd4j.rand(DataType.FLOAT, 4*nOut));

    double fb = 1.0;
    LSTMConfiguration conf = LSTMConfiguration.builder()
            .peepHole(true)
            .forgetBias(fb)
            .clippingCellValue(0.0)
            .build();

    LSTMWeights weights = LSTMWeights.builder().weights(W).bias(b)
            .inputPeepholeWeights(Wci).forgetPeepholeWeights(Wcf).outputPeepholeWeights(Wco).build();

    LSTMCellOutputs v = new LSTMCellOutputs(sd.rnn().lstmCell(x, cLast, yLast, weights, conf));  //Output order: i, c, f, o, z, h, y
    List<String> toExec = new ArrayList<>();
    for(SDVariable sdv : v.getAllOutputs()){
        toExec.add(sdv.name());
    }

    //Test forward pass:
    Map<String,INDArray> m = sd.output(null, toExec);

    //Weights and bias order: [i, f, z, o]

    //Block input (z) - post tanh:
    INDArray wz_x = W.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(nOut, 2*nOut));           //Input weights
    INDArray wz_r = W.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(nOut, 2*nOut));    //Recurrent weights
    INDArray bz = b.getArr().get(NDArrayIndex.interval(nOut, 2*nOut));

    INDArray zExp = x.getArr().mmul(wz_x).addiRowVector(bz);        //[mb,nIn]*[nIn, nOut] + [nOut]
    zExp.addi(yLast.getArr().mmul(wz_r));   //[mb,nOut]*[nOut,nOut]
    Transforms.tanh(zExp, false);

    INDArray zAct = m.get(toExec.get(4));
    assertEquals(zExp, zAct);

    //Input modulation gate (post sigmoid) - i: (note: peephole input - last time step)
    INDArray wi_x = W.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(0, nOut));           //Input weights
    INDArray wi_r = W.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(0, nOut));    //Recurrent weights
    INDArray bi = b.getArr().get(NDArrayIndex.interval(0, nOut));

    INDArray iExp = x.getArr().mmul(wi_x).addiRowVector(bi);        //[mb,nIn]*[nIn, nOut] + [nOut]
    iExp.addi(yLast.getArr().mmul(wi_r));   //[mb,nOut]*[nOut,nOut]
    iExp.addi(cLast.getArr().mulRowVector(Wci.getArr()));    //Peephole
    Transforms.sigmoid(iExp, false);
    assertEquals(iExp, m.get(toExec.get(0)));

    //Forget gate (post sigmoid): (note: peephole input - last time step)
    INDArray wf_x = W.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(2*nOut, 3*nOut));           //Input weights
    INDArray wf_r = W.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(2*nOut, 3*nOut));    //Recurrent weights
    INDArray bf = b.getArr().get(NDArrayIndex.interval(2*nOut, 3*nOut));

    INDArray fExp = x.getArr().mmul(wf_x).addiRowVector(bf);        //[mb,nIn]*[nIn, nOut] + [nOut]
    fExp.addi(yLast.getArr().mmul(wf_r));   //[mb,nOut]*[nOut,nOut]
    fExp.addi(cLast.getArr().mulRowVector(Wcf.getArr()));   //Peephole
    fExp.addi(fb);
    Transforms.sigmoid(fExp, false);
    assertEquals(fExp, m.get(toExec.get(2)));

    //Cell state (pre tanh): tanh(z) .* sigmoid(i) + sigmoid(f) .* cLast
    INDArray cExp = zExp.mul(iExp).add(fExp.mul(cLast.getArr()));
    INDArray cAct = m.get(toExec.get(1));
    assertEquals(cExp, cAct);

    //Output gate (post sigmoid): (note: peephole input: current time step)
    INDArray wo_x = W.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(3*nOut, 4*nOut));           //Input weights
    INDArray wo_r = W.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(3*nOut, 4*nOut));    //Recurrent weights
    INDArray bo = b.getArr().get(NDArrayIndex.interval(3*nOut, 4*nOut));

    INDArray oExp = x.getArr().mmul(wo_x).addiRowVector(bo);        //[mb,nIn]*[nIn, nOut] + [nOut]
    oExp.addi(yLast.getArr().mmul(wo_r));   //[mb,nOut]*[nOut,nOut]
    oExp.addi(cExp.mulRowVector(Wco.getArr())); //Peephole
    Transforms.sigmoid(oExp, false);
    assertEquals(oExp, m.get(toExec.get(3)));

    //Cell state, post tanh
    INDArray hExp = Transforms.tanh(cExp, true);
    assertEquals(hExp, m.get(toExec.get(5)));

    //Final output
    INDArray yExp = hExp.mul(oExp);
    assertEquals(yExp, m.get(toExec.get(6)));
}
 

@Test
public void testGRUCell(){
    Nd4j.getRandom().setSeed(12345);
    int mb = 2;
    int nIn = 3;
    int nOut = 4;

    SameDiff sd = SameDiff.create();
    SDVariable x = sd.constant(Nd4j.rand(DataType.FLOAT, mb, nIn));
    SDVariable hLast = sd.constant(Nd4j.rand(DataType.FLOAT, mb, nOut));
    SDVariable Wru = sd.constant(Nd4j.rand(DataType.FLOAT, (nIn+nOut), 2*nOut));
    SDVariable Wc = sd.constant(Nd4j.rand(DataType.FLOAT, (nIn+nOut), nOut));
    SDVariable bru = sd.constant(Nd4j.rand(DataType.FLOAT, 2*nOut));
    SDVariable bc = sd.constant(Nd4j.rand(DataType.FLOAT, nOut));

    double fb = 1.0;
    GRUWeights weights = GRUWeights.builder()
            .ruWeight(Wru)
            .cWeight(Wc)
            .ruBias(bru)
            .cBias(bc)
            .build();

    SDVariable[] v = sd.rnn().gruCell(x, hLast, weights);
    List<String> toExec = new ArrayList<>();
    for(SDVariable sdv : v){
        toExec.add(sdv.name());
    }

    //Test forward pass:
    Map<String,INDArray> m = sd.output(null, toExec);

    //Weights and bias order: [r, u], [c]

    //Reset gate:
    INDArray wr_x = Wru.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(0, nOut));           //Input weights
    INDArray wr_r = Wru.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(0, nOut));    //Recurrent weights
    INDArray br = bru.getArr().get(NDArrayIndex.interval(0, nOut));

    INDArray rExp = x.getArr().mmul(wr_x).addiRowVector(br);        //[mb,nIn]*[nIn, nOut] + [nOut]
    rExp.addi(hLast.getArr().mmul(wr_r));   //[mb,nOut]*[nOut,nOut]
    Transforms.sigmoid(rExp,false);

    INDArray rAct = m.get(toExec.get(0));
    assertEquals(rExp, rAct);

    //Update gate:
    INDArray wu_x = Wru.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.interval(nOut, 2*nOut));           //Input weights
    INDArray wu_r = Wru.getArr().get(NDArrayIndex.interval(nIn,nIn+nOut), NDArrayIndex.interval(nOut, 2*nOut));    //Recurrent weights
    INDArray bu = bru.getArr().get(NDArrayIndex.interval(nOut, 2*nOut));

    INDArray uExp = x.getArr().mmul(wu_x).addiRowVector(bu);        //[mb,nIn]*[nIn, nOut] + [nOut]
    uExp.addi(hLast.getArr().mmul(wu_r));   //[mb,nOut]*[nOut,nOut]
    Transforms.sigmoid(uExp,false);

    INDArray uAct = m.get(toExec.get(1));
    assertEquals(uExp, uAct);

    //c = tanh(x * Wcx + Wcr * (hLast .* r))
    INDArray Wcx = Wc.getArr().get(NDArrayIndex.interval(0,nIn), NDArrayIndex.all());
    INDArray Wcr = Wc.getArr().get(NDArrayIndex.interval(nIn, nIn+nOut), NDArrayIndex.all());
    INDArray cExp = x.getArr().mmul(Wcx);
    cExp.addi(hLast.getArr().mul(rExp).mmul(Wcr));
    cExp.addiRowVector(bc.getArr());
    Transforms.tanh(cExp, false);

    assertEquals(cExp, m.get(toExec.get(2)));

    //h = u * hLast + (1-u) * c
    INDArray hExp = uExp.mul(hLast.getArr()).add(uExp.rsub(1.0).mul(cExp));
    assertEquals(hExp, m.get(toExec.get(3)));
}
 

public static INDArray activate(@NonNull INDArray boundingBoxPriors, @NonNull INDArray input, boolean nchw, LayerWorkspaceMgr layerWorkspaceMgr){
    if(!nchw)
        input = input.permute(0,3,1,2); //NHWC to NCHW

    long mb = input.size(0);
    long h = input.size(2);
    long w = input.size(3);
    long b = boundingBoxPriors.size(0);
    long c = input.size(1)/b-5;  //input.size(1) == b * (5 + C) -> C = (input.size(1)/b) - 5

    INDArray output = layerWorkspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), input.shape(), 'c');
    INDArray output5 = output.reshape('c', mb, b, 5+c, h, w);
    INDArray output4 = output;  //output.get(all(), interval(0,5*b), all(), all());
    INDArray input4 = input.dup('c');    //input.get(all(), interval(0,5*b), all(), all()).dup('c');
    INDArray input5 = input4.reshape('c', mb, b, 5+c, h, w);

    //X/Y center in grid: sigmoid
    INDArray predictedXYCenterGrid = input5.get(all(), all(), interval(0,2), all(), all());
    Transforms.sigmoid(predictedXYCenterGrid, false);

    //width/height: prior * exp(input)
    INDArray predictedWHPreExp = input5.get(all(), all(), interval(2,4), all(), all());
    INDArray predictedWH = Transforms.exp(predictedWHPreExp, false);
    Broadcast.mul(predictedWH, boundingBoxPriors.castTo(input.dataType()), predictedWH, 1, 2);  //Box priors: [b, 2]; predictedWH: [mb, b, 2, h, w]

    //Confidence - sigmoid
    INDArray predictedConf = input5.get(all(), all(), point(4), all(), all());   //Shape: [mb, B, H, W]
    Transforms.sigmoid(predictedConf, false);

    output4.assign(input4);

    //Softmax
    //TODO OPTIMIZE?
    INDArray inputClassesPreSoftmax = input5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    INDArray classPredictionsPreSoftmax2d = inputClassesPreSoftmax.permute(0,1,3,4,2) //[minibatch, b, c, h, w] To [mb, b, h, w, c]
            .dup('c').reshape('c', new long[]{mb*b*h*w, c});
    Transforms.softmax(classPredictionsPreSoftmax2d, false);
    INDArray postSoftmax5d = classPredictionsPreSoftmax2d.reshape('c', mb, b, h, w, c ).permute(0, 1, 4, 2, 3);

    INDArray outputClasses = output5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    outputClasses.assign(postSoftmax5d);

    if(!nchw)
        output = output.permute(0,2,3,1);       //NCHW to NHWC

    return output;
}
 

@Test
public void testBernoulliLogProb() {
    Nd4j.getRandom().setSeed(12345);

    int inputSize = 4;
    int[] mbs = new int[] {1, 2, 5};

    Random r = new Random(12345);

    for (boolean average : new boolean[] {true, false}) {
        for (int minibatch : mbs) {

            INDArray x = Nd4j.zeros(minibatch, inputSize);
            for (int i = 0; i < minibatch; i++) {
                for (int j = 0; j < inputSize; j++) {
                    x.putScalar(i, j, r.nextInt(2));
                }
            }

            INDArray distributionParams = Nd4j.rand(minibatch, inputSize).muli(2).subi(1); //i.e., pre-sigmoid prob
            INDArray prob = Transforms.sigmoid(distributionParams, true);

            ReconstructionDistribution dist = new BernoulliReconstructionDistribution(Activation.SIGMOID);

            double negLogProb = dist.negLogProbability(x, distributionParams, average);

            INDArray exampleNegLogProb = dist.exampleNegLogProbability(x, distributionParams);
            assertArrayEquals(new long[] {minibatch, 1}, exampleNegLogProb.shape());

            //Calculate the same thing, but using Apache Commons math

            double logProbSum = 0.0;
            for (int i = 0; i < minibatch; i++) {
                double exampleSum = 0.0;
                for (int j = 0; j < inputSize; j++) {
                    double p = prob.getDouble(i, j);

                    BinomialDistribution binomial = new BinomialDistribution(1, p); //Bernoulli is a special case of binomial

                    double xVal = x.getDouble(i, j);
                    double thisLogProb = binomial.logProbability((int) xVal);
                    logProbSum += thisLogProb;
                    exampleSum += thisLogProb;
                }
                assertEquals(-exampleNegLogProb.getDouble(i), exampleSum, 1e-6);
            }

            double expNegLogProb;
            if (average) {
                expNegLogProb = -logProbSum / minibatch;
            } else {
                expNegLogProb = -logProbSum;
            }

            //                System.out.println(x);

            //                System.out.println(expNegLogProb + "\t" + logProb + "\t" + (logProb / expNegLogProb));
            assertEquals(expNegLogProb, negLogProb, 1e-6);

            //Also: check random sampling...
            int count = minibatch * inputSize;
            INDArray arr = Nd4j.linspace(-3, 3, count, Nd4j.dataType()).reshape(minibatch, inputSize);
            INDArray sampleMean = dist.generateAtMean(arr);
            INDArray sampleRandom = dist.generateRandom(arr);

            for (int i = 0; i < minibatch; i++) {
                for (int j = 0; j < inputSize; j++) {
                    double d1 = sampleMean.getDouble(i, j);
                    double d2 = sampleRandom.getDouble(i, j);
                    assertTrue(d1 >= 0.0 || d1 <= 1.0); //Mean value - probability... could do 0 or 1 (based on most likely) but that isn't very useful...
                    assertTrue(d2 == 0.0 || d2 == 1.0);
                }
            }
        }
    }
}
 

public static INDArray doSigmoid(INDArray input) {
    return Transforms.sigmoid(input, true);
}