Java Code Examples for org.nd4j.autodiff.samediff.SameDiff#setLossVariables()

@Test
public void testGather2(){
    SameDiff sd = SameDiff.create();
    SDVariable input = sd.var("in", Nd4j.arange(6).castTo(DataType.FLOAT).reshape(2,3));
    SDVariable indices = sd.constant("indices", Nd4j.createFromArray(0));

    SDVariable gathered = sd.gather(input, indices, 1);
    SDVariable loss = gathered.std(true);

    sd.output((Map<String,INDArray>)null, gathered.name());
    sd.setLossVariables(gathered.name());

    String err = OpValidation.validate(new TestCase(sd)
            .gradCheckEpsilon(1e-3)
            .gradCheckMaxRelativeError(1e-4));

    assertNull(err);
}
 

@Test
public void testMaxPooling3dBasic() {
    int nIn = 3;
    int kH = 2;
    int kW = 2;
    int kD = 2;

    int mb = 3;
    int imgH = 5;
    int imgW = 5;
    int imgD = 5;

    SameDiff sd = SameDiff.create();
    INDArray inArr = Nd4j.create(mb, nIn, imgD, imgH, imgW);

    SDVariable in = sd.var("in", inArr);

    Pooling3DConfig pooling3DConfig = Pooling3DConfig.builder()
            .kH(kH).kW(kW).kD(kD)
            .pH(0).pW(0).pD(0)
            .sH(1).sW(1).sD(1)
            .dH(1).dW(1).dD(1)
            .isSameMode(false)
            .build();

    SDVariable out = sd.cnn().maxPooling3d(in, pooling3DConfig);
    out = sd.nn().tanh("loss", out).shape().rename("out");

    sd.setLossVariables("loss");

    // oH = (iH - (kH + (kH-1)*(dH-1)) + 2*pH)/sH + 1;
    INDArray outArr = Nd4j.createFromArray(mb, nIn, 4, 4, 4L);

    TestCase tc = new TestCase(sd).expectedOutput("out", outArr).gradientCheck(false);
    String err = OpValidation.validate(tc);
    assertNull(err);
}
 

@Test
public void testConv1dBasic() {
    int nIn = 3;
    int nOut = 4;
    int k = 2;
    int mb = 3;
    int img = 28;

    SameDiff sd = SameDiff.create();
    INDArray wArr = Nd4j.create(k, nIn, nOut);
    INDArray inArr = Nd4j.create(mb, nIn, img);

    SDVariable in = sd.var("in", inArr);
    SDVariable w = sd.var("W", wArr);

    SDVariable[] vars = new SDVariable[]{in, w};

    Conv1DConfig conv1DConfig = Conv1DConfig.builder()
            .k(k).p(0).s(1)
            .paddingMode(PaddingMode.VALID)
            .build();

    SDVariable out = sd.cnn().conv1d(in, w, conv1DConfig);
    out = sd.nn().tanh("loss", out).shape().rename("out");

    sd.setLossVariables("loss");

    //Expected output size: out = (in - k + 2*p)/s + 1 = (28-2+0)/1+1 = 27
    INDArray outArr = Nd4j.createFromArray(mb, nOut, 27L);
    TestCase tc = new TestCase(sd).expectedOutput("out", outArr).gradientCheck(false);
    String err = OpValidation
            .validate(tc);
    assertNull(err);
}
 

@Test
public void GRUTestCase() {
    int bS = 5;
    int nIn = 4;
    int nOut = 6;
    int time = 2;

    SameDiff sd = SameDiff.create();

    SDVariable in = sd.var("in", Nd4j.randn(DataType.DOUBLE, time, bS, nIn).muli(10));
    SDVariable hLast = sd.var("cLast", Nd4j.zeros(DataType.DOUBLE, bS, nOut));
    SDVariable Wx = sd.var("Wx", Nd4j.randn(DataType.DOUBLE, nIn, 3*nOut));
    SDVariable Wh = sd.var("Wh", Nd4j.randn(DataType.DOUBLE, nOut, 3*nOut));
    SDVariable biases = sd.var("bias", Nd4j.randn(DataType.DOUBLE, 3*nOut));

    SDVariable out = new GRU(sd, in, hLast, Wx, Wh,biases).outputVariable();

    long[] outShapes = new long[]{time,bS, nOut};
    assertArrayEquals(new long[]{time,bS, nOut}, out.eval().shape());

    sd.setLossVariables(out.std(true));
    String err = OpValidation.validate(new TestCase(sd)
            .gradientCheck(true)
    );

    assertNull(err);

}
 

@Test
public void testSoftmaxCrossEntropyWithLogitsLoss() {
    OpValidationSuite.ignoreFailing();

    SameDiff sameDiff = SameDiff.create();

    INDArray labels = Nd4j.createFromArray(new double[]{
            0,1,1,0,0,0,1,0,1,0,1,1,1,0,1,0,1,0,0,1,1,0,1,0
    }).reshape(2,3,4);

    INDArray logits = Nd4j.linspace(DataType.DOUBLE, 0.1, 0.1, 24).reshape(2,3,4);
    INDArray expected = Nd4j.createFromArray(new double[]{
            0.26328, 1.46328, 1.72656, 0.     , 0.26328, 0.     , 1.46328, 0.26328, 1.72656, 0.     , 1.72656, 1.46328
    }).reshape(3,4);

    SDVariable sdLogits = sameDiff.var("logits", logits);
    SDVariable sdLabels = sameDiff.var("labels", labels);
    SDVariable loss = sameDiff.math().abs(sdLogits);


    SDVariable output = new SoftmaxCrossEntropyWithLogitsLoss(sameDiff, sdLogits, sdLabels, 0).outputVariable();
    sameDiff.setLossVariables(output);

    TestCase tc = new TestCase(sameDiff)
            .gradientCheck(true)
            .expectedOutput(output.name(), expected);

    String err = OpValidation.validate(tc);
    assertNull(err);
}
 

@Test
public void testConv1dCausal() {
    Nd4j.getRandom().setSeed(12345);
    int nIn = 3;
    int nOut = 4;
    int mb = 2;

    for (int k : new int[]{2, 3}) {
        for (int sz : new int[]{3, 4, 5}) {
            for (int s : new int[]{1, 2}) {
                for (int d : new int[]{1, 2}) {
                    for (boolean ncw : new boolean[]{true, false}) {

                        SameDiff sd = SameDiff.create();
                        INDArray wArr = Nd4j.rand(DataType.DOUBLE, k, nIn, nOut);
                        INDArray inArr = Nd4j.rand(DataType.DOUBLE, (ncw ? new long[]{mb, nIn, sz} : new long[]{mb, sz, nIn}));
                        INDArray bArr = Nd4j.rand(DataType.DOUBLE, nOut);

                        SDVariable in = sd.var("in", inArr);
                        SDVariable w = sd.var("W", wArr);
                        SDVariable b = sd.var("b", bArr);

                        Conv1DConfig conv1DConfig = Conv1DConfig.builder()
                                .dataFormat(ncw ? Conv1DConfig.NCW : Conv1DConfig.NWC)
                                .k(k).p(0).s(s).d(d)
                                .paddingMode(PaddingMode.CAUSAL)
                                .build();

                        SDVariable out = sd.cnn().conv1d(in, w, b, conv1DConfig);
                        SDVariable loss = sd.nn().tanh(out).std(true).rename("loss");

                        sd.setLossVariables("loss");

                        String name = "k=" + k + ", sz=" + sz + ", ncw=" + ncw;

                        System.out.println(name);

                        TestCase tc = new TestCase(sd).testName(name).gradientCheck(true);
                        String err = OpValidation
                                .validate(tc);
                        assertNull(err);
                    }
                }
            }
        }
    }
}
 

@Test
public void testConv3dBasic() {
    int nIn = 3;
    int nOut = 4;
    int kH = 2;
    int kW = 2;
    int kD = 2;

    int mb = 3;
    int imgH = 5;
    int imgW = 5;
    int imgT = 5;

    SameDiff sd = SameDiff.create();
    INDArray wArr = Nd4j.rand(new int[]{kD, kH, kW, nIn, nOut});
    INDArray bArr = Nd4j.rand(1, nOut);
    INDArray inArr = Nd4j.rand(new int[]{mb, nIn, imgT, imgH, imgW});

    SDVariable in = sd.var("in", inArr);
    SDVariable w = sd.var("W", wArr);
    SDVariable b = sd.var("b", bArr);

    Conv3DConfig conv3DConfig = Conv3DConfig.builder()
            .kH(kH).kW(kW).kD(kD)
            .sD(1).sH(1).sW(1)
            .dH(1).dW(1).dD(1)
            .isSameMode(true)
            .biasUsed(false)
            .dataFormat(Conv3DConfig.NCDHW)
            .build();

    SDVariable out = sd.cnn().conv3d(in, w, b, conv3DConfig);
    out = sd.nn().tanh("loss", out).shape().rename("out");

    sd.setLossVariables("loss");

    //Expected output size, NOT same mode: out = (in - k)/d + 1 = (28-2+0)/1+1 = 27
    //Expected output size, WITH same mode: out = in/stride
    INDArray outArr = Nd4j.createFromArray(mb, nOut, 5, 5, 5L);

    TestCase tc = new TestCase(sd).expectedOutput("out", outArr).gradientCheck(true);
    String err = OpValidation
            .validate(tc);
    assertNull(err);
}
 

@Test
public void testDeConv3dBasic() {
    int nIn = 4;
    int nOut = 3;
    int kH = 2;
    int kW = 2;
    int kD = 2;

    int mb = 3;
    int imgH = 5;
    int imgW = 5;
    int imgT = 5;

    SameDiff sd = SameDiff.create();
    INDArray inArr = Nd4j.rand(new long[]{mb, nIn, 5, 5, 5});
    INDArray wArr = Nd4j.rand(kD, kH, kW, nOut, nIn);

    SDVariable in = sd.var("in", inArr);
    SDVariable w = sd.var("W", wArr);

    DeConv3DConfig conv3DConfig = DeConv3DConfig.builder()
            .kH(kH).kW(kW).kD(kD)
            .sD(1).sH(1).sW(1)
            .dH(1).dW(1).dD(1)
            .isSameMode(true)
            .dataFormat(DeConv3DConfig.NCDHW)
            .build();

    SDVariable out = sd.cnn().deconv3d(in, w, conv3DConfig);
    out = sd.nn().tanh("loss", out).shape().rename("out");

    sd.setLossVariables("loss");

    //Expected conv3d size, NOT same mode: out = (in - k)/d + 1 = (28-2+0)/1+1 = 27
    //Expected conv3d size, WITH same mode: out = in/stride
    // reversed this for deconv3d
    INDArray outArr = Nd4j.createFromArray(new long[]{mb, nOut, imgT, imgH, imgW});

    TestCase tc = new TestCase(sd)
            .expectedOutput("out", outArr)
            .gradientCheck(true);
    String err = OpValidation.validate(tc);
    assertNull(err);
}
 

@Test
public void LSTMLayerTestCase1() {

    int bS = 5;
    int nIn = 3;
    int numUnits = 7;
    int sL = 3; //small just for test


    // notations:
    // bS - batch size, numExamples
    // sL - sequence length, number of time steps, timeLength
    // nIn - input size, inOutSize

    //  TNS: shape [timeLength, numExamples, inOutSize] - sometimes referred to as "time major"<br>
    //  NST: shape [numExamples, inOutSize, timeLength]<br>
    //  NTS: shape [numExamples, timeLength, inOutSize]<br>
    //  for bidirectional:
    //  T2NS: 3 = [timeLength, 2, numExamples, inOutSize] (for ONNX)


    for (boolean useCLast : new boolean[]{false, true}) {
        for (boolean useYLast : new boolean[]{false, true}) {

            SameDiff sd = SameDiff.create();
            SDVariable in = sd.var("in", Nd4j.randn(DataType.DOUBLE, bS, nIn, sL));


            SDVariable cLast = useCLast ? sd.var("cLast", Nd4j.zeros(DataType.DOUBLE, bS, numUnits)) : null;
            SDVariable yLast = useYLast ? sd.var("yLast", Nd4j.zeros(DataType.DOUBLE, bS, numUnits)) : null;


            LSTMLayerConfig c = LSTMLayerConfig.builder()
                    .lstmdataformat(LSTMDataFormat.NST)
                    .directionMode(LSTMDirectionMode.FWD)
                    .gateAct(LSTMActivations.SIGMOID)
                    .cellAct(LSTMActivations.TANH)
                    .outAct(LSTMActivations.TANH)
                    .retFullSequence(true)
                    .retLastC(true)
                    .retLastH(true)
                    .build();

            LSTMLayerOutputs outputs = new LSTMLayerOutputs(sd.rnn.lstmLayer(
                    in, cLast, yLast, null,
                    LSTMLayerWeights.builder()
                            .weights(sd.var("weights", Nd4j.randn(DataType.DOUBLE, nIn, 4 * numUnits)))
                            .rWeights(sd.var("rWeights", Nd4j.randn(DataType.DOUBLE, numUnits, 4 * numUnits)))
                            .peepholeWeights(sd.var("inputPeepholeWeights", Nd4j.randn(DataType.DOUBLE, 3 * numUnits)))
                            .bias(sd.var("bias", Nd4j.rand(DataType.DOUBLE, 4 * numUnits))).build(),
                    c), c);

            long[] out = new long[]{bS, numUnits, sL};
            long[] hL = new long[]{bS, numUnits};
            long[] cL = new long[]{bS, numUnits};

            assertArrayEquals(out, outputs.getOutput().eval().shape());
            assertArrayEquals(hL, outputs.getLastOutput().eval().shape());
            assertArrayEquals(cL, outputs.getLastState().eval().shape());

            sd.setLossVariables(outputs.getOutput(), outputs.getLastTimeStepOutput(), outputs.getTimeSeriesOutput());

            String err = OpValidation.validate(new TestCase(sd)
                    .gradientCheck(true)
                    .testName("cLast=" + cLast + ", yLast=" + yLast)
            );

            assertNull(err);
        }
    }


}
 

@Test
public void LSTMLayerTestCase2() {
    int bS = 5;
    int nIn = 3;
    int numUnits = 7;
    int sL = 3; //small just for test

    SameDiff sd = SameDiff.create();

    // notations:
    // bS - batch size, numExamples
    // sL - sequence length, number of time steps, timeLength
    // nIn - input size, inOutSize

    //  TNS: shape [timeLength, numExamples, inOutSize] - sometimes referred to as "time major"<br>
    //  NST: shape [numExamples, inOutSize, timeLength]<br>
    //  NTS: shape [numExamples, timeLength, inOutSize]<br>
    //  for bidirectional:
    //  T2NS: 3 = [timeLength, 2, numExamples, inOutSize] (for ONNX)
    SDVariable in = sd.var("in", Nd4j.rand(DataType.DOUBLE, sL, bS, nIn));


    SDVariable cLast = sd.var("cLast", Nd4j.zeros(DataType.DOUBLE, bS, numUnits));
    SDVariable yLast = sd.var("yLast", Nd4j.zeros(DataType.DOUBLE, bS, numUnits));

    LSTMLayerConfig c = LSTMLayerConfig.builder()
            .lstmdataformat(LSTMDataFormat.TNS)
            .directionMode(LSTMDirectionMode.FWD)
            .gateAct(LSTMActivations.SIGMOID)
            .cellAct(LSTMActivations.TANH)
            .outAct(LSTMActivations.TANH)
            .retFullSequence(true)
            .retLastC(false)
            .retLastH(false)
            .build();

    LSTMLayerOutputs outputs = new LSTMLayerOutputs(sd.rnn.lstmLayer(
            in, cLast, yLast, null,
            LSTMLayerWeights.builder()
                    .weights(sd.var("weights", Nd4j.rand(DataType.DOUBLE, nIn, 4 * numUnits)))
                    .rWeights(sd.var("rWeights", Nd4j.rand(DataType.DOUBLE, numUnits, 4 * numUnits)))
                    .build(),
            c), c);


    long[] out = new long[]{sL, bS, numUnits};
    assertArrayEquals(out, outputs.getOutput().eval().shape());

    sd.setLossVariables(outputs.getOutput());

    String err = OpValidation.validate(new TestCase(sd)
            .gradientCheck(true)
    );

    assertNull(err);

}
 

@Test
public void LSTMLayerTestCase3() {
    int bS = 5;
    int nIn = 3;
    int numUnits = 7;
    int sL = 3; //small just for test

    SameDiff sd = SameDiff.create();

    // notations:
    // bS - batch size, numExamples
    // sL - sequence length, number of time steps, timeLength
    // nIn - input size, inOutSize

    //  TNS: shape [timeLength, numExamples, inOutSize] - sometimes referred to as "time major"<br>
    //  NST: shape [numExamples, inOutSize, timeLength]<br>
    //  NTS: shape [numExamples, timeLength, inOutSize]<br>
    //  for bidirectional:
    //  T2NS: 3 = [timeLength, 2, numExamples, inOutSize] (for ONNX)
    SDVariable in = sd.var("in", Nd4j.rand(DataType.DOUBLE, bS, sL, nIn));


    // when directionMode >= 2 (BIDIR_CONCAT=3)
    // Wx, Wr [2, nIn, 4*nOut]
    // hI, cI [2, bS, nOut]
    SDVariable cLast = sd.var("cLast", Nd4j.zeros(DataType.DOUBLE, 2, bS, numUnits));
    SDVariable yLast = sd.var("yLast", Nd4j.zeros(DataType.DOUBLE, 2, bS, numUnits));

    LSTMLayerConfig c = LSTMLayerConfig.builder()
            .lstmdataformat(LSTMDataFormat.NTS)
            .directionMode(LSTMDirectionMode.BIDIR_CONCAT)
            .gateAct(LSTMActivations.SIGMOID)
            .cellAct(LSTMActivations.SOFTPLUS)
            .outAct(LSTMActivations.SOFTPLUS)
            .retFullSequence(true)
            .retLastC(false)
            .retLastH(false)
            .build();

    LSTMLayerOutputs outputs = new LSTMLayerOutputs(sd.rnn.lstmLayer(new String[]{"out"},
            in, cLast, yLast, null,
            LSTMLayerWeights.builder()
                    .weights(sd.var("weights", Nd4j.rand(DataType.DOUBLE, 2, nIn, 4 * numUnits)))
                    .rWeights(sd.var("rWeights", Nd4j.rand(DataType.DOUBLE, 2, numUnits, 4 * numUnits)))
                    .build(),
            c), c);


    long[] out = new long[]{bS, sL, 2 * numUnits};

    assertArrayEquals(out, outputs.getOutput().eval().shape());

    sd.setLossVariables(outputs.getOutput());

    String err = OpValidation.validate(new TestCase(sd)
            .gradientCheck(true)
    );

    assertNull(err);
}