Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#putScalar()

private static INDArray yoloLabels(int mb, int c, int h, int w){
    int labelDepth = 4 + c;
    INDArray labels = Nd4j.zeros(mb, labelDepth, h, w);
    //put 1 object per minibatch, at positions (0,0), (1,1) etc.
    //Positions for label boxes: (1,1) to (2,2), (2,2) to (4,4) etc

    for( int i=0; i<mb; i++ ){
        //Class labels
        labels.putScalar(i, 4 + i%c, i%h, i%w, 1);

        //BB coordinates (top left, bottom right)
        labels.putScalar(i, 0, 0, 0, i%w);
        labels.putScalar(i, 1, 0, 0, i%h);
        labels.putScalar(i, 2, 0, 0, (i%w)+1);
        labels.putScalar(i, 3, 0, 0, (i%h)+1);
    }

    return labels;
}
 

public static boolean checkMulManually(INDArray first, INDArray second, double maxRelativeDifference,
                double minAbsDifference) {
    //No apache commons element-wise multiply, but can do this manually

    INDArray result = first.mul(second);
    long[] shape = first.shape();

    INDArray expected = Nd4j.zeros(first.shape());

    for (int i = 0; i < shape[0]; i++) {
        for (int j = 0; j < shape[1]; j++) {
            double v = first.getDouble(i, j) * second.getDouble(i, j);
            expected.putScalar(new int[] {i, j}, v);
        }
    }
    if (!checkShape(expected, result))
        return false;
    boolean ok = checkEntries(expected, result, maxRelativeDifference, minAbsDifference);
    if (!ok) {
        INDArray onCopies = Shape.toOffsetZeroCopy(first).mul(Shape.toOffsetZeroCopy(second));
        printFailureDetails(first, second, expected, result, onCopies, "mul");
    }
    return ok;
}
 

@Test
public void testTanhDerivative() {

    //Derivative of sigmoid: ds(x)/dx = s(x)*(1-s(x))
    //s(x) = 1 / (exp(-x) + 1)
    INDArray z = Nd4j.zeros(100);
    double[] expOut = new double[100];
    for (int i = 0; i < 100; i++) {
        double x = 0.1 * (i - 50);
        z.putScalar(i, x);
        double tanh = FastMath.tanh(x);
        expOut[i] = 1.0 - tanh * tanh;
    }

    INDArray zPrime = Nd4j.getExecutioner().exec(new TanhDerivative(z));

    for (int i = 0; i < 100; i++) {
        double relError = Math.abs(expOut[i] - zPrime.getDouble(i))
                        / (Math.abs(expOut[i]) + Math.abs(zPrime.getDouble(i)));
        assertTrue(relError < REL_ERROR_TOLERANCE);
    }
}
 

@Override
public INDArray transform(List<String> tokens) {
    INDArray ret = Nd4j.create(1, vocabCache.numWords());

    Map<String, AtomicLong> counts = new HashMap<>();
    for (String token : tokens) {
        if (!counts.containsKey(token))
            counts.put(token, new AtomicLong(0));

        counts.get(token).incrementAndGet();
    }

    for (int i = 0; i < tokens.size(); i++) {
        int idx = vocabCache.indexOf(tokens.get(i));
        if (idx >= 0) {
            double tf_idf = tfidfWord(tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size());
            //log.info("TF-IDF for word: {} -> {} / {} => {}", tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size(), tf_idf);
            ret.putScalar(idx, tf_idf);
        }
    }
    return ret;
}
 

/**
 * Returns a column vector where each entry is the nth bilinear
 * product of the nth slices of the two tensors.
 */
@Override
public INDArray bilinearProducts(INDArray curr, INDArray in) {
    Preconditions.checkArgument(curr.rank() == 3, "Argument 'curr' must be rank 3. Got input with rank: %s", curr.rank());
    if (in.columns() != 1) {
        throw new AssertionError("Expected a column vector");
    }
    if (in.rows() != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Number of rows in the input does not match number of columns in tensor");
    }
    if (curr.size(curr.shape().length - 2) != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices");
    }

    INDArray ret = Nd4j.create(curr.slices(), 1);
    INDArray inT = in.transpose();
    for (int i = 0; i < curr.slices(); i++) {
        INDArray slice = curr.slice(i);
        INDArray inTTimesSlice = inT.mmul(slice);
        ret.putScalar(i, Nd4j.getBlasWrapper().dot(inTTimesSlice, in));
    }
    return ret;
}
 

public static INDArray fastBitmapToMatrix(final FastBitmap bitmap, final ColorSpace colorSpace) {
    INDArray result;

    double[][][] bitmapMatrix;

    switch (colorSpace) {
        case Grayscale:
            result = Nd4j.create(bitmap.toMatrixGrayAsDouble());
            break;
        case ARGB:
            result = Nd4j.create(bitmap.getWidth(), bitmap.getHeight(), 4);
            bitmapMatrix = bitmap.toMatrixRGBAsDouble();

            for (int i = 0; i < bitmap.getWidth(); i++) {
                for (int j = 0; j < bitmap.getHeight(); j++) {
                    result.putScalar(new int[]{i, j, 0}, bitmapMatrix[j][i][0]);
                    result.putScalar(new int[]{i, j, 1}, bitmapMatrix[j][i][1]);
                    result.putScalar(new int[]{i, j, 2}, bitmapMatrix[j][i][2]);
                    result.putScalar(new int[]{i, j, 3}, bitmapMatrix[j][i][3]);
                }
            }
            break;
        case RGB:
            bitmapMatrix = bitmap.toMatrixRGBAsDouble();

            result = Nd4j.create(bitmap.getWidth(), bitmap.getHeight(), 3);
            for (int i = 0; i < bitmap.getWidth(); i++) {
                for (int j = 0; j < bitmap.getHeight(); j++) {
                    result.putScalar(new int[]{i, j, 0}, bitmapMatrix[j][i][0]);
                    result.putScalar(new int[]{i, j, 1}, bitmapMatrix[j][i][1]);
                    result.putScalar(new int[]{i, j, 2}, bitmapMatrix[j][i][2]);
                }
            }
            break;
        default:
            logger.warn("Could not determine color space. Saving an empty matrix...");
            result = Nd4j.create(new double[]{});
    }
    return result;
}
 

@Test
public void testCompareRocAndRocMultiClass() {
    Nd4j.getRandom().setSeed(12345);

    //For 2 class case: ROC and Multi-class ROC should be the same...
    int nExamples = 200;
    INDArray predictions = Nd4j.rand(nExamples, 2);
    INDArray tempSum = predictions.sum(1);
    predictions.diviColumnVector(tempSum);

    INDArray labels = Nd4j.create(nExamples, 2);
    Random r = new Random(12345);
    for (int i = 0; i < nExamples; i++) {
        labels.putScalar(i, r.nextInt(2), 1.0);
    }

    for (int numSteps : new int[] {30, 0}) { //Steps = 0: exact
        ROC roc = new ROC(numSteps);
        roc.eval(labels, predictions);

        ROCMultiClass rocMultiClass = new ROCMultiClass(numSteps);
        rocMultiClass.eval(labels, predictions);

        double auc = roc.calculateAUC();
        double auc1 = rocMultiClass.calculateAUC(1);

        assertEquals(auc, auc1, 1e-6);
    }
}
 

/**
 * Copy real numbers to arr
 * @param arr the arr to copy to
 */
protected void copyRealTo(INDArray arr) {
    INDArray linear = arr.linearView();
    IComplexNDArray thisLinear = linearView();
    if (arr.isScalar())
        arr.putScalar(0, getReal(0));
    else
        for (int i = 0; i < linear.length(); i++) {
            arr.putScalar(i, thisLinear.getReal(i));
        }

}
 

@Test
public void testNDArrayWritablesMultilabel() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 0);
    arr3.putScalar(1, 1);
    arr3.putScalar(2, 0);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
            new NDArrayWritable(arr2),
            new IntWritable(2),
            new DoubleWritable(3),
            new NDArrayWritable(arr3),
            new DoubleWritable(1));
    File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0";

    try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}
 

/**
 * Get the search candidates as indices given the input
 * and similarity function
 * @param x the input data to search with
 * @param trees the trees to search
 * @param similarityFunction the function to use for similarity
 * @return the list of indices as the search results
 */
public static INDArray getAllCandidates(INDArray x,List<RPTree> trees,String similarityFunction) {
    List<Integer> candidates = getCandidates(x,trees,similarityFunction);
    Collections.sort(candidates);

    int prevIdx = -1;
    int idxCount = 0;
    List<Pair<Integer,Integer>> scores = new ArrayList<>();
    for(int i = 0; i < candidates.size(); i++) {
        if(candidates.get(i) == prevIdx) {
            idxCount++;
        }
        else if(prevIdx != -1) {
            scores.add(Pair.of(idxCount,prevIdx));
            idxCount = 1;
        }

        prevIdx = i;
    }


    scores.add(Pair.of(idxCount,prevIdx));

    INDArray arr = Nd4j.create(scores.size());
    for(int i = 0; i < scores.size(); i++) {
        arr.putScalar(i,scores.get(i).getSecond());
    }

    return arr;
}
 

public static INDArray convertFromApacheMatrix(RealMatrix matrix) {
    int[] shape = new int[] {matrix.getRowDimension(), matrix.getColumnDimension()};
    INDArray out = Nd4j.create(shape);
    for (int i = 0; i < shape[0]; i++) {
        for (int j = 0; j < shape[1]; j++) {
            double value = matrix.getEntry(i, j);
            out.putScalar(new int[] {i, j}, value);
        }
    }
    return out;
}
 

@Test
public void when_colorsAreConverted_expect_grayScaleResult() {
    // Assign
    int numChannels = 3;
    int height = 1;
    int width = 5;

    RGBtoGrayscaleDataSetPreProcessor sut = new RGBtoGrayscaleDataSetPreProcessor();
    INDArray input = Nd4j.create(2, numChannels, height, width);

    // Black, Example 1
    input.putScalar(0, 0, 0, 0, 0.0 );
    input.putScalar(0, 1, 0, 0, 0.0 );
    input.putScalar(0, 2, 0, 0, 0.0 );

    // White, Example 1
    input.putScalar(0, 0, 0, 1, 255.0 );
    input.putScalar(0, 1, 0, 1, 255.0 );
    input.putScalar(0, 2, 0, 1, 255.0 );

    // Red, Example 1
    input.putScalar(0, 0, 0, 2, 255.0 );
    input.putScalar(0, 1, 0, 2, 0.0 );
    input.putScalar(0, 2, 0, 2, 0.0 );

    // Green, Example 1
    input.putScalar(0, 0, 0, 3, 0.0 );
    input.putScalar(0, 1, 0, 3, 255.0 );
    input.putScalar(0, 2, 0, 3, 0.0 );

    // Blue, Example 1
    input.putScalar(0, 0, 0, 4, 0.0 );
    input.putScalar(0, 1, 0, 4, 0.0 );
    input.putScalar(0, 2, 0, 4, 255.0 );


    // Black, Example 2
    input.putScalar(1, 0, 0, 4, 0.0 );
    input.putScalar(1, 1, 0, 4, 0.0 );
    input.putScalar(1, 2, 0, 4, 0.0 );

    // White, Example 2
    input.putScalar(1, 0, 0, 3, 255.0 );
    input.putScalar(1, 1, 0, 3, 255.0 );
    input.putScalar(1, 2, 0, 3, 255.0 );

    // Red, Example 2
    input.putScalar(1, 0, 0, 2, 255.0 );
    input.putScalar(1, 1, 0, 2, 0.0 );
    input.putScalar(1, 2, 0, 2, 0.0 );

    // Green, Example 2
    input.putScalar(1, 0, 0, 1, 0.0 );
    input.putScalar(1, 1, 0, 1, 255.0 );
    input.putScalar(1, 2, 0, 1, 0.0 );

    // Blue, Example 2
    input.putScalar(1, 0, 0, 0, 0.0 );
    input.putScalar(1, 1, 0, 0, 0.0 );
    input.putScalar(1, 2, 0, 0, 255.0 );

    DataSet ds = new DataSet(input, null);

    // Act
    sut.preProcess(ds);

    // Assert
    INDArray result = ds.getFeatures();
    long[] shape = result.shape();

    assertEquals(3, shape.length);
    assertEquals(2, shape[0]);
    assertEquals(1, shape[1]);
    assertEquals(5, shape[2]);

    assertEquals(0.0, result.getDouble(0, 0, 0), 0.05);
    assertEquals(255.0, result.getDouble(0, 0, 1), 0.05);
    assertEquals(255.0 * 0.3, result.getDouble(0, 0, 2), 0.05);
    assertEquals(255.0 * 0.59, result.getDouble(0, 0, 3), 0.05);
    assertEquals(255.0 * 0.11, result.getDouble(0, 0, 4), 0.05);

    assertEquals(0.0, result.getDouble(1, 0, 4), 0.05);
    assertEquals(255.0, result.getDouble(1, 0, 3), 0.05);
    assertEquals(255.0 * 0.3, result.getDouble(1, 0, 2), 0.05);
    assertEquals(255.0 * 0.59, result.getDouble(1, 0, 1), 0.05);
    assertEquals(255.0 * 0.11, result.getDouble(1, 0, 0), 0.05);

}
 

@Test
public void testGradientApplyMultiLayerNetwork() {
    int minibatch = 7;
    int nIn = 10;
    int nOut = 10;

    for (boolean regularization : new boolean[] {false, true}) {
        for (IUpdater u : new IUpdater[] {new Sgd(0.1), new Nesterovs(0.1), new Adam(0.1)}) {

            MultiLayerConfiguration conf =
                            new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH)
                                            .weightInit(WeightInit.XAVIER).updater(u)
                                            .l1(regularization ? 0.2 : 0.0)
                                            .l2(regularization ? 0.3 : 0.0).list()
                                            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(10).build())
                                            .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2,
                                                            new OutputLayer.Builder(
                                                                            LossFunctions.LossFunction.MCXENT)
                                                                                            .activation(Activation.SOFTMAX)
                                                                                            .nIn(10).nOut(nOut)
                                                                                            .build())
                                            .build();


            Nd4j.getRandom().setSeed(12345);
            MultiLayerNetwork net1GradCalc = new MultiLayerNetwork(conf);
            net1GradCalc.init();

            Nd4j.getRandom().setSeed(12345);
            MultiLayerNetwork net2GradUpd = new MultiLayerNetwork(conf.clone());
            net2GradUpd.init();

            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            INDArray f = Nd4j.rand(minibatch, nIn);
            INDArray l = Nd4j.create(minibatch, nOut);
            for (int i = 0; i < minibatch; i++) {
                l.putScalar(i, i % nOut, 1.0);
            }
            net1GradCalc.setInput(f);
            net1GradCalc.setLabels(l);

            net2GradUpd.setInput(f);
            net2GradUpd.setLabels(l);

            //Calculate gradient in first net, update and apply it in the second
            //Also: calculate gradient in the second net, just to be sure it isn't modified while doing updating on
            // the other net's gradient
            net1GradCalc.computeGradientAndScore();
            net2GradUpd.computeGradientAndScore();

            Gradient g = net1GradCalc.gradient();
            INDArray gBefore = g.gradient().dup(); //Net 1 gradient should be modified
            INDArray net2GradBefore = net2GradUpd.gradient().gradient().dup(); //But net 2 gradient should not be
            net2GradUpd.getUpdater().update(net2GradUpd, g, 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces());
            INDArray gAfter = g.gradient().dup();
            INDArray net2GradAfter = net2GradUpd.gradient().gradient().dup();

            assertNotEquals(gBefore, gAfter); //Net 1 gradient should be modified
            assertEquals(net2GradBefore, net2GradAfter); //But net 2 gradient should not be


            //Also: if we apply the gradient using a subi op, we should get the same final params as if we did a fit op
            // on the original network
            net2GradUpd.params().subi(g.gradient());

            net1GradCalc.fit(f, l);
            assertEquals(net1GradCalc.params(), net2GradUpd.params());


            //=============================
            if (!(u instanceof Sgd)) {
                net2GradUpd.getUpdater().getStateViewArray().assign(net1GradCalc.getUpdater().getStateViewArray());
            }
            assertEquals(net1GradCalc.params(), net2GradUpd.params());
            assertEquals(net1GradCalc.getUpdater().getStateViewArray(),
                            net2GradUpd.getUpdater().getStateViewArray());

            //Remove the next 2 lines: fails - as net 1 is 1 iteration ahead
            net1GradCalc.getLayerWiseConfigurations().setIterationCount(0);
            net2GradUpd.getLayerWiseConfigurations().setIterationCount(0);

            for (int i = 0; i < 100; i++) {
                net1GradCalc.fit(f, l);
                net2GradUpd.fit(f, l);
                assertEquals(net1GradCalc.params(), net2GradUpd.params());
            }
        }
    }
}
 

@Test
public void testGemmWithOpsCommonsMath() {
    List<Pair<INDArray, String>> first = NDArrayCreationUtil.getAllTestMatricesWithShape(3, 5, SEED);
    List<Pair<INDArray, String>> firstT = NDArrayCreationUtil.getAllTestMatricesWithShape(5, 3, SEED);
    List<Pair<INDArray, String>> second = NDArrayCreationUtil.getAllTestMatricesWithShape(5, 4, SEED);
    List<Pair<INDArray, String>> secondT = NDArrayCreationUtil.getAllTestMatricesWithShape(4, 5, SEED);
    double[] alpha = {1.0, -0.5, 2.5};
    double[] beta = {0.0, -0.25, 1.5};
    INDArray cOrig = Nd4j.create(new int[] {3, 4});
    Random r = new Random(12345);
    for (int i = 0; i < cOrig.size(0); i++) {
        for (int j = 0; j < cOrig.size(1); j++) {
            cOrig.putScalar(new int[] {i, j}, r.nextDouble());
        }
    }

    for (int i = 0; i < first.size(); i++) {
        for (int j = 0; j < second.size(); j++) {
            for (int k = 0; k < alpha.length; k++) {
                for (int m = 0; m < beta.length; m++) {
                    System.out.println((String.format("Running iteration %d %d %d %d", i, j, k, m)));

                    INDArray cff = Nd4j.create(cOrig.shape(), 'f');
                    cff.assign(cOrig);
                    INDArray cft = Nd4j.create(cOrig.shape(), 'f');
                    cft.assign(cOrig);
                    INDArray ctf = Nd4j.create(cOrig.shape(), 'f');
                    ctf.assign(cOrig);
                    INDArray ctt = Nd4j.create(cOrig.shape(), 'f');
                    ctt.assign(cOrig);

                    double a = alpha[k];
                    double b = beta[k];
                    Pair<INDArray, String> p1 = first.get(i);
                    Pair<INDArray, String> p1T = firstT.get(i);
                    Pair<INDArray, String> p2 = second.get(j);
                    Pair<INDArray, String> p2T = secondT.get(j);
                    String errorMsgff = getGemmErrorMsg(i, j, false, false, a, b, p1, p2);
                    String errorMsgft = getGemmErrorMsg(i, j, false, true, a, b, p1, p2T);
                    String errorMsgtf = getGemmErrorMsg(i, j, true, false, a, b, p1T, p2);
                    String errorMsgtt = getGemmErrorMsg(i, j, true, true, a, b, p1T, p2T);

                    assertTrue(errorMsgff, CheckUtil.checkGemm(p1.getFirst(), p2.getFirst(), cff, false, false, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgft, CheckUtil.checkGemm(p1.getFirst(), p2T.getFirst(), cft, false, true, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgtf, CheckUtil.checkGemm(p1T.getFirst(), p2.getFirst(), ctf, true, false, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgtt, CheckUtil.checkGemm(p1T.getFirst(), p2T.getFirst(), ctt, true, true, a,
                                    b, 1e-4, 1e-6));
                }
            }
        }
    }
}
 

@Test
    public void testCnnWithSpaceToBatch() {
        Nd4j.getRandom().setSeed(12345);
        int nOut = 4;

        int[] minibatchSizes = {2, 4};
        int width = 5;
        int height = 5;
        int inputDepth = 1;

        int[] kernel = {2, 2};
        int[] blocks = {1, 1};

        String[] activations = {"sigmoid", "tanh"};
        SubsamplingLayer.PoolingType[] poolingTypes =
                new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

        for (String afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                    for (int i = 0; i < minibatchSize; i++) {
                        labels.putScalar(new int[]{i, i % nOut}, 1.0);
                    }

                    MultiLayerConfiguration conf =
                            new NeuralNetConfiguration.Builder()
                                    .dataType(DataType.DOUBLE)
                                    .updater(new NoOp())
                                    .dist(new NormalDistribution(0, 1))
                                    .list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth)
                                    .cudnnAllowFallback(false)
                                    .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                    .layer(new SpaceToBatchLayer.Builder(blocks).build()) //trivial space to batch
                                    .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                            .activation(Activation.SOFTMAX).nIn(4 * 4 * 3)
                                            .nOut(nOut).build())
                                    .setInputType(InputType.convolutionalFlat(height, width, inputDepth))
                                    .build();

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                            + afn;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
//                        for (int j = 0; j < net.getnLayers(); j++)
//                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }
                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);
                }
            }
        }
    }
 

@Test
    public void testSeparableConv2D() {
        int nOut = 2;

        int[] minibatchSizes = new int[]{1, 3};
        int width = 6;
        int height = 6;
        int inputDepth = 3;

        Nd4j.getRandom().setSeed(12345);

        int[] ks = new int[]{1, 3, 3, 1, 3};
        int[] ss = new int[]{1, 1, 1, 2, 2};
        int[] ds = new int[]{1, 1, 2, 2, 2};
        ConvolutionMode[] cms = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate};
        int[] mb = new int[]{1, 1, 1, 3, 3};

        for (int t = 0; t < ks.length; t++) {

            int k = ks[t];
            int s = ss[t];
            int d = ds[t];
            ConvolutionMode cm = cms[t];
            int minibatchSize = mb[t];

            //Use larger input with larger dilation values (to avoid invalid config)
            int w = d * width;
            int h = d * height;

            INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp())
                    .activation(Activation.TANH)
                    .convolutionMode(cm)
                    .list()
                    .layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer")
                            .kernelSize(k, k)
                            .stride(s, s)
                            .dilation(d, d)
                            .depthMultiplier(3)
                            .nIn(inputDepth).nOut(2).build());

            MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

//            for (int i = 0; i < net.getLayers().length; i++) {
//                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
//            }

            String msg = " - mb=" + minibatchSize + ", k="
                    + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(
                    new GradientCheckUtil.MLNConfig().net(net)
                            .input(input).labels(labels)
                            .subset(true).maxPerParam(50));

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }
 

@Test
    public void testGradientBNWithCNNandSubsamplingCompGraph() {
        //Parameterized test, testing combinations of:
        // (a) activation function
        // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
        // (c) Loss function (with specified output activations)
        // (d) l1 and l2 values
        Activation[] activFns = {Activation.TANH, Activation.IDENTITY};
        boolean doLearningFirst = true;

        LossFunctions.LossFunction[] lossFunctions = {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD};
        Activation[] outputActivations = {Activation.SOFTMAX}; //i.e., lossFunctions[i] used with outputActivations[i] here

        double[] l2vals = {0.0, 0.1};
        double[] l1vals = {0.0, 0.2}; //i.e., use l2vals[j] with l1vals[j]

        Nd4j.getRandom().setSeed(12345);
        int minibatch = 10;
        int depth = 2;
        int hw = 5;
        int nOut = 3;
        INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw});
        INDArray labels = Nd4j.zeros(minibatch, nOut);
        Random r = new Random(12345);
        for (int i = 0; i < minibatch; i++) {
            labels.putScalar(i, r.nextInt(nOut), 1.0);
        }

        DataSet ds = new DataSet(input, labels);

        for (boolean useLogStd : new boolean[]{true, false}) {
            for (Activation afn : activFns) {
                for (int i = 0; i < lossFunctions.length; i++) {
                    for (int j = 0; j < l2vals.length; j++) {
                        LossFunctions.LossFunction lf = lossFunctions[i];
                        Activation outputActivation = outputActivations[i];

                        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                                .dataType(DataType.DOUBLE)
                                .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
                                .updater(new NoOp())
                                .dist(new UniformDistribution(-2, 2)).seed(12345L).graphBuilder()
                                .addInputs("in")
                                .addLayer("0", new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3)
                                        .activation(afn).build(), "in")
                                .addLayer("1", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "0")
                                .addLayer("2", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                        .kernelSize(2, 2).stride(1, 1).build(), "1")
                                .addLayer("3", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "2")
                                .addLayer("4", new ActivationLayer.Builder().activation(afn).build(), "3")
                                .addLayer("5", new OutputLayer.Builder(lf).activation(outputActivation)
                                        .nOut(nOut).build(), "4")
                                .setOutputs("5").setInputTypes(InputType.convolutional(hw, hw, depth))
                                .build();

                        ComputationGraph net = new ComputationGraph(conf);
                        net.init();
                        String name = new Object() {
                        }.getClass().getEnclosingMethod().getName();

                        if (doLearningFirst) {
                            //Run a number of iterations of learning
                            net.setInput(0, ds.getFeatures());
                            net.setLabels(ds.getLabels());
                            net.computeGradientAndScore();
                            double scoreBefore = net.score();
                            for (int k = 0; k < 20; k++)
                                net.fit(ds);
                            net.computeGradientAndScore();
                            double scoreAfter = net.score();
                            //Can't test in 'characteristic mode of operation' if not learning
                            String msg = name
                                    + " - score did not (sufficiently) decrease during learning - activationFn="
                                    + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                                    + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                                    + ", scoreAfter=" + scoreAfter + ")";
                            assertTrue(msg, scoreAfter < 0.9 * scoreBefore);
                        }

                        System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf
                                + ", outputActivation=" + outputActivation + ", doLearningFirst="
                                + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]);
//                        for (int k = 0; k < net.getNumLayers(); k++)
//                            System.out.println("Layer " + k + " # params: " + net.getLayer(k).numParams());

                        //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
                        //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
                        //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
                        Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev"));
                        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{input})
                                .labels(new INDArray[]{labels}).excludeParams(excludeParams));

                        assertTrue(gradOK);
                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 

private DataSet vectorize(List<? extends Sample> aData, Object2IntMap<String> aTagset,
        boolean aIncludeLabels)
    throws IOException
{
    // vectorize is pretty fast taking around 1-2ms
    
    // long start = System.currentTimeMillis();
    int maxSentenceLength = traits.getMaxSentenceLength();
    
    // Create data for training
    int embeddingSize = wordVectors.dimensions(); 
    INDArray featureVec = Nd4j.create(aData.size(), embeddingSize, maxSentenceLength);

    // Tags are using a 1-hot encoding
    INDArray labelVec = Nd4j.create(aData.size(), traits.getMaxTagsetSize(), maxSentenceLength);
    
    // Sentences have variable length, so we we need to mask positions not used in short
    // sentences.
    INDArray featureMask = Nd4j.zeros(aData.size(), maxSentenceLength);
    INDArray labelMask = Nd4j.zeros(aData.size(), maxSentenceLength);
    
    // Get word vectors for each word in review, and put them in the training data
    int sampleIdx = 0;
    for (Sample sample : aData) {
        List<String> tokens = sample.getSentence();
        List<String> labels = sample.getTags();
        for (int t = 0; t < Math.min(tokens.size(), maxSentenceLength); t++) {
            String word = tokens.get(t);
            INDArray vector = Nd4j.create(wordVectors.vectorize(word));

            if (vector == null) {
                vector = randUnk;
            }

            featureVec.put(new INDArrayIndex[] { point(sampleIdx), all(), point(t) }, vector);
            featureMask.putScalar(new int[] { sampleIdx, t }, 1.0);

            // exclude padding labels from training
            // compare instances to avoid collision with possible no_label user label
            if (labels != null && labels.get(t) != NO_LABEL) {
                labelMask.putScalar(new int[] { sampleIdx, t }, 1.0);
            }

            if (aIncludeLabels && labels != null) {
                String label = labels.get(t);
                // do not add padding label no_label as predictable label
                if (label != NO_LABEL) {
                    aTagset.computeIfAbsent(label, key -> aTagset.size());
                    labelVec.putScalar(sampleIdx, aTagset.get(label), t, 1.0);
                }
            }
        }
        
        sampleIdx++;
    }

    // log.trace("Vectorizing took {}ms", System.currentTimeMillis() - start);
    
    return new DataSet(featureVec, labelVec, featureMask, labelMask);
}
 

@Test
public void testCapsNetDtypes() {
    for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
        Nd4j.setDefaultDataTypes(globalDtype, globalDtype);
        for (DataType networkDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
            assertEquals(globalDtype, Nd4j.dataType());
            assertEquals(globalDtype, Nd4j.defaultFloatingPointType());

            String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype;

            int primaryCapsDim = 2;
            int primarpCapsChannel = 8;
            int capsule = 5;
            int minibatchSize = 8;
            int routing = 1;
            int capsuleDim = 4;
            int height = 6;
            int width = 6;
            int inputDepth = 4;

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .dataType(networkDtype)
                    .seed(123)
                    .updater(new NoOp())
                    .weightInit(new WeightInitDistribution(new UniformDistribution(-6, 6)))
                    .list()
                    .layer(new PrimaryCapsules.Builder(primaryCapsDim, primarpCapsChannel)
                            .kernelSize(3, 3)
                            .stride(2, 2)
                            .build())
                    .layer(new CapsuleLayer.Builder(capsule, capsuleDim, routing).build())
                    .layer(new CapsuleStrengthLayer.Builder().build())
                    .layer(new ActivationLayer.Builder(new ActivationSoftmax()).build())
                    .layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build())
                    .setInputType(InputType.convolutional(height, width, inputDepth))
                    .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray in = Nd4j.rand(networkDtype, minibatchSize, inputDepth * height * width).mul(10)
                    .reshape(-1, inputDepth, height, width);
            INDArray label = Nd4j.zeros(networkDtype, minibatchSize, capsule);
            for (int i = 0; i < minibatchSize; i++) {
                label.putScalar(new int[]{i, i % capsule}, 1.0);
            }

            INDArray out = net.output(in);
            assertEquals(msg, networkDtype, out.dataType());
            List<INDArray> ff = net.feedForward(in);
            for (int i = 0; i < ff.size(); i++) {
                String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName());
                assertEquals(s, networkDtype, ff.get(i).dataType());
            }

            net.setInput(in);
            net.setLabels(label);
            net.computeGradientAndScore();

            net.fit(new DataSet(in, label));

            logUsedClasses(net);

            //Now, test mismatched dtypes for input/labels:
            for (DataType inputLabelDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
                INDArray in2 = in.castTo(inputLabelDtype);
                INDArray label2 = label.castTo(inputLabelDtype);
                net.output(in2);
                net.setInput(in2);
                net.setLabels(label2);
                net.computeGradientAndScore();

                net.fit(new DataSet(in2, label2));
            }
        }
    }
}
 

@Test
public void testLearnedSelfAttentionLayer_differentMiniBatchSizes() {
    int nIn = 3;
    int nOut = 2;
    int tsLength = 4;
    int layerSize = 4;
    int numQueries = 3;

    Random r = new Random(12345);
    for (boolean inputMask : new boolean[]{false, true}) {
        for (boolean projectInput : new boolean[]{false, true}) {

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .dataType(DataType.DOUBLE)
                .activation(Activation.TANH)
                .updater(new NoOp())
                .weightInit(WeightInit.XAVIER)
                .list()
                .layer(new LSTM.Builder().nOut(layerSize).build())
                .layer( projectInput ?
                        new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build()
                        : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build()
                )
                .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build())
                .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(nIn))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        for (int mb : new int[]{3, 1}) {
                INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength});
                INDArray labels = TestUtils.randomOneHot(mb, nOut);
                String maskType = (inputMask ? "inputMask" : "none");

                INDArray inMask = null;
                if (inputMask) {
                    inMask = Nd4j.ones(DataType.INT, mb, tsLength);
                    for (int i = 0; i < mb; i++) {
                        int firstMaskedStep = tsLength - 1 - i;
                        if (firstMaskedStep == 0) {
                            firstMaskedStep = tsLength;
                        }
                        for (int j = firstMaskedStep; j < tsLength; j++) {
                            inMask.putScalar(i, j, 0.0);
                        }
                    }
                }

                String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput;
                System.out.println("Starting test: " + name);

                boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in)
                        .labels(labels).inputMask(inMask).subset(true).maxPerParam(100));
                assertTrue(name, gradOK);
            }
        }
    }
}