Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#permute()

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = input.dup('f');
    //Input: 3d activations (RNN)
    //Output: 4d activations (CNN)
    if (rnnDataFormat == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    val shape = input.shape();
    INDArray in2d;
    if (shape[0] == 1) {
        //Edge case: miniBatchSize = 1
        in2d = input.tensorAlongDimension(0, 1, 2).permutei(1, 0);
    } else if (shape[2] == 1) {
        //Edge case: time series length = 1
        in2d = input.tensorAlongDimension(0, 1, 0);
    } else {
        INDArray permuted = input.permute(0, 2, 1); //Permute, so we get correct order after reshaping
        in2d = permuted.reshape('f', shape[0] * shape[2], shape[1]);
    }

    return workspaceMgr.dup(ArrayType.ACTIVATIONS, in2d, 'c')
            .reshape('c', shape[0] * shape[2], numChannels, inputHeight, inputWidth);
}
 

@Override
public Image asImageMatrix(InputStream inputStream, boolean nchw) throws IOException {
    Mat mat = streamToMat(inputStream);
    Mat image = imdecode(mat, IMREAD_ANYDEPTH | IMREAD_ANYCOLOR);
    if (image == null || image.empty()) {
        PIX pix = pixReadMem(mat.data(), mat.cols());
        if (pix == null) {
            throw new IOException("Could not decode image from input stream");
        }
        image = convert(pix);
        pixDestroy(pix);
    }
    INDArray a = asMatrix(image);
    if(!nchw)
        a = a.permute(0,2,3,1);     //NCHW to NHWC
    Image i = new Image(a, image.channels(), image.rows(), image.cols());

    image.deallocate();
    return i;
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (output == null)
        return null; //In a few cases: output may be null, and this is valid. Like time series data -> embedding layer
    //Need to reshape FeedForward layer epsilons (2d) to 3d (for use in RNN layer backprop calculations)
    if (output.rank() != 2)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 2 (i.e., epsilons from feed forward layer)");
    if (output.ordering() != 'f' || !Shape.hasDefaultStridesForShape(output))
        output = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, output, 'f');

    val shape = output.shape();
    INDArray reshaped = output.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    if (rnnDataFormat == RNNFormat.NCW){
        reshaped = reshaped.permute(0, 2, 1);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, reshaped);
}
 

@Test
public void testNoShape1() {
    int outDepth = 50;
    int miniBatch = 64;
    int outH = 8;
    int outW = 8;

    try (Nd4jWorkspace wsI =
                    (Nd4jWorkspace) Nd4j.getWorkspaceManager().getAndActivateWorkspace(basicConfig, "ITER")) {
        INDArray delta = Nd4j.create(new int[] {50, 64, 8, 8}, new int[] {64, 3200, 8, 1}, 'c');
        delta = delta.permute(1, 0, 2, 3);

        assertArrayEquals(new long[] {64, 50, 8, 8}, delta.shape());
        assertArrayEquals(new long[] {3200, 64, 8, 1}, delta.stride());

        INDArray delta2d = Shape.newShapeNoCopy(delta, new int[] {outDepth, miniBatch * outH * outW}, false);

        assertNotNull(delta2d);
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);

    INDArray input = this.input.castTo(dataType);   //Cast to network dtype if required (no-op if already correct type)

    boolean nchw = layerConf().getFormat() == CNN2DFormat.NCHW;

    INDArray outEpsilon = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), input.shape(), 'c');

    Gradient gradient = new DefaultGradient();

    INDArray epsilonNHWC = nchw ? epsilon.permute(0, 2, 3, 1) : epsilon;
    INDArray outEpsilonNHWC = nchw ? outEpsilon.permute(0, 2, 3, 1) : outEpsilon;

    CustomOp op = DynamicCustomOp.builder("batch_to_space_nd")
            .addInputs(epsilonNHWC, getBlocksArray(), getPaddingArray())
            .addOutputs(outEpsilonNHWC)
            .callInplace(false)
            .build();
    Nd4j.exec(op);

    outEpsilon = backpropDropOutIfPresent(outEpsilon);
    return new Pair<>(gradient, outEpsilon);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Need to reshape FF activations (2d) activations to 3d (for input into RNN layer)
    if (input.rank() != 2)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 2 (i.e., activations for FF layer)");
    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'f');

    val shape = input.shape();
    INDArray reshaped = input.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    if (rnnDataFormat == RNNFormat.NCW){
        reshaped = reshaped.permute(0, 2, 1);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, reshaped);
}
 

@Test
public void testFlattenedView() {
    int rows = 8;
    int cols = 8;
    int dim2 = 4;
    int length = rows * cols;
    int length3d = rows * cols * dim2;

    INDArray first = Nd4j.linspace(1, length, length).reshape('c', rows, cols);
    INDArray second = Nd4j.create(new int[]{rows, cols}, 'f').assign(first);
    INDArray third = Nd4j.linspace(1, length3d, length3d).reshape('c', rows, cols, dim2);
    first.addi(0.1);
    second.addi(0.2);
    third.addi(0.3);

    first = first.get(NDArrayIndex.interval(4, 8), NDArrayIndex.interval(0, 2, 8));
    for (int i = 0; i < first.tensorssAlongDimension(0); i++) {
        System.out.println(first.tensorAlongDimension(i, 0));
    }

    for (int i = 0; i < first.tensorssAlongDimension(1); i++) {
        System.out.println(first.tensorAlongDimension(i, 1));
    }
    second = second.get(NDArrayIndex.interval(3, 7), NDArrayIndex.all());
    third = third.permute(0, 2, 1);

    INDArray cAssertion = Nd4j.create(new double[]{33.10, 35.10, 37.10, 39.10, 41.10, 43.10, 45.10, 47.10, 49.10,
            51.10, 53.10, 55.10, 57.10, 59.10, 61.10, 63.10});
    INDArray fAssertion = Nd4j.create(new double[]{33.10, 41.10, 49.10, 57.10, 35.10, 43.10, 51.10, 59.10, 37.10,
            45.10, 53.10, 61.10, 39.10, 47.10, 55.10, 63.10});
    assertEquals(cAssertion, Nd4j.toFlattened('c', first));
    assertEquals(fAssertion, Nd4j.toFlattened('f', first));
}
 

@Test
public void cnnBatchNormTest() throws IOException, UnsupportedKerasConfigurationException,
        InvalidKerasConfigurationException {

    String modelPath = "modelimport/keras/fullconfigs/cnn/cnn_batch_norm.h5";

    KerasSequentialModel kerasModel = new KerasModel().modelBuilder()
            .modelHdf5Filename(Resources.asFile(modelPath).getAbsolutePath())
            .enforceTrainingConfig(false)
            .buildSequential();

    MultiLayerNetwork model = kerasModel.getMultiLayerNetwork();
    model.init();

    System.out.println(model.summary());

    INDArray input = Nd4j.createFromNpyFile(Resources.asFile("modelimport/keras/fullconfigs/cnn/input.npy"));
    input = input.permute(0, 3, 1, 2);
    assertTrue(Arrays.equals(input.shape(), new long[] {5, 3, 10, 10}));

    INDArray output = model.output(input);

    INDArray kerasOutput = Nd4j.createFromNpyFile(Resources.asFile("modelimport/keras/fullconfigs/cnn/predictions.npy"));

    for (int i = 0; i < 5; i++) {
        TestCase.assertEquals(output.getDouble(i), kerasOutput.getDouble(i), 1e-4);
    }
}
 

@Test
public void testPermuteReshape() {
    INDArray arrTest = Nd4j.arange(60).reshape('c', 3, 4, 5);
    INDArray permute = arrTest.permute(2, 1, 0);
    assertArrayEquals(new long[] {5, 4, 3}, permute.shape());
    assertArrayEquals(new long[] {1, 5, 20}, permute.stride());
    INDArray reshapedPermute = permute.reshape(-1, 12);
    assertArrayEquals(new long[] {5, 12}, reshapedPermute.shape());
    assertArrayEquals(new long[] {12, 1}, reshapedPermute.stride());

}
 

private void setMaskFromInput(INDArray input) {
    if (input.rank() != 3) {
        throw new IllegalArgumentException("Expected input of shape [batch_size, timestep_input_size, timestep], " +
                "got shape "+Arrays.toString(input.shape()) + " instead");
    }
    if ((underlying instanceof BaseRecurrentLayer &&
            ((BaseRecurrentLayer)underlying).getDataFormat() == NWC)){
        input = input.permute(0, 2, 1);
    }
    INDArray mask = input.eq(maskingValue).castTo(input.dataType()).sum(1).neq(input.shape()[1]).castTo(input.dataType());
    underlying.setMaskArray(mask.detach());
}
 

/**
 * Functions which reduces the dimensionality of the given <code>dataset</code>
 * by applying max pooling with kernel size and stride of 8.
 *
 * @param dataset Dataset to be reduced in place
 */
public static void reduceHighDimensionalityByPoolingInPlace(final DataSet dataset) {
	logger.info("Applying max pooling on input data due to high dimensionality.. Final result calculation will be done on original dimensionality.");

	long[] shape = dataset.getIntermediateInstances().get(0).shape();
	long[] resultingShape = null;
	MultiLayerNetwork mln = ImageUtils.getMaxPoolNetworkSymmetricWithCustomKernelStride(8, 8);

	boolean permute = shape.length > 2 && shape[2] > 1;
	for (int i = 0; i < dataset.getIntermediateInstances().size(); i++) {
		INDArray matrix = dataset.getIntermediateInstances().get(i);
		if (permute) {
			matrix = matrix.permute(2, 0, 1);
		}

		if (shape.length > 2) {
			matrix = matrix.reshape(1, shape[2], shape[0], shape[1]);
		} else {
			matrix = matrix.reshape(1, 1, shape[0], shape[1]);
		}
		matrix = ImageUtils.applyMLNToMatrix(matrix, mln);

		if (resultingShape == null) {
			resultingShape = matrix.shape();
		}

		// Reverse
		if (permute) {
			matrix = matrix.permute(0, 2, 3, 1);
		}
		if (shape.length > 2) {
			matrix = matrix.reshape(resultingShape[2], resultingShape[3], resultingShape[1]);
		} else {
			matrix = matrix.reshape(resultingShape[2], resultingShape[3]);
		}

		dataset.getIntermediateInstances().set(i, matrix);
	}
	logger.info("Applied max pooling.");
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray input = this.input;
    if (input.rank() != 3)
        throw new UnsupportedOperationException(
                        "Input must be rank 3. Got input with rank " + input.rank() + " " + layerId());
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);

    applyDropOutIfNecessary(training, workspaceMgr);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    INDArray input2d = TimeSeriesUtils.reshape3dTo2d(input.castTo(W.dataType()), workspaceMgr, ArrayType.FF_WORKING_MEM);

    INDArray act2d = layerConf().getActivationFn().getActivation(input2d.mmul(W).addiRowVector(b), training);
    if (maskArray != null) {
        if(!maskArray.isColumnVectorOrScalar() || Arrays.equals(maskArray.shape(), act2d.shape())){
            //Per output masking
            act2d.muli(maskArray.castTo(act2d.dataType()));
        } else {
            //Per time step masking
            act2d.muliColumnVector(maskArray.castTo(act2d.dataType()));
        }
    }

    INDArray ret = TimeSeriesUtils.reshape2dTo3d(act2d, input.size(0), workspaceMgr, ArrayType.ACTIVATIONS);
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        ret = ret.permute(0, 2, 1);
    }
    return ret;
}
 

@Test
public void cnnBatchNormLargerTest() throws IOException, UnsupportedKerasConfigurationException,
        InvalidKerasConfigurationException {

    String modelPath = "modelimport/keras/fullconfigs/cnn_batch_norm/cnn_batch_norm_medium.h5";

    KerasSequentialModel kerasModel = new KerasModel().modelBuilder()
            .modelHdf5Filename(Resources.asFile(modelPath).getAbsolutePath())
            .enforceTrainingConfig(false)
            .buildSequential();

    MultiLayerNetwork model = kerasModel.getMultiLayerNetwork();
    model.init();

    System.out.println(model.summary());

    INDArray input = Nd4j.createFromNpyFile(Resources.asFile("modelimport/keras/fullconfigs/cnn_batch_norm/input.npy"));
    input = input.permute(0, 3, 1, 2);
    assertTrue(Arrays.equals(input.shape(), new long[] {5, 1, 48, 48}));

    INDArray output = model.output(input);

    INDArray kerasOutput = Nd4j.createFromNpyFile(Resources.asFile("modelimport/keras/fullconfigs/cnn_batch_norm/predictions.npy"));

    for (int i = 0; i < 5; i++) {
        // TODO this should be a little closer
        TestCase.assertEquals(output.getDouble(i), kerasOutput.getDouble(i), 1e-2);
    }
}
 

@Test
public void testPermuteReshape() {
    INDArray arrTest = Nd4j.arange(60).reshape('c', 3, 4, 5);
    INDArray permute = arrTest.permute(2, 1, 0);
    assertArrayEquals(new long[] {5, 4, 3}, permute.shape());
    assertArrayEquals(new long[] {1, 5, 20}, permute.stride());
    INDArray reshapedPermute = permute.reshape(-1, 12);
    assertArrayEquals(new long[] {5, 12}, reshapedPermute.shape());
    assertArrayEquals(new long[] {12, 1}, reshapedPermute.stride());

}
 

/**
 * Set weights for layer.
 *
 * @param weights Map of weights
 */
@Override
public void setWeights(Map<String, INDArray> weights) throws InvalidKerasConfigurationException {
    this.weights = new HashMap<>();

    INDArray dW;
    if (weights.containsKey(conf.getLAYER_PARAM_NAME_DEPTH_WISE_KERNEL())) {
        dW = weights.get(conf.getLAYER_PARAM_NAME_DEPTH_WISE_KERNEL());
        dW = dW.permute(3, 2, 0, 1);
    } else
        throw new InvalidKerasConfigurationException(
                "Keras SeparableConvolution2D layer does not contain parameter "
                        + conf.getLAYER_PARAM_NAME_DEPTH_WISE_KERNEL());

    this.weights.put(SeparableConvolutionParamInitializer.DEPTH_WISE_WEIGHT_KEY, dW);

    INDArray pW;
    if (weights.containsKey(conf.getLAYER_PARAM_NAME_POINT_WISE_KERNEL())) {
        pW = weights.get(conf.getLAYER_PARAM_NAME_POINT_WISE_KERNEL());
        pW = pW.permute(3, 2, 0, 1);
    }
    else
        throw new InvalidKerasConfigurationException(
                "Keras SeparableConvolution2D layer does not contain parameter "
                        + conf.getLAYER_PARAM_NAME_POINT_WISE_KERNEL());

    this.weights.put(SeparableConvolutionParamInitializer.POINT_WISE_WEIGHT_KEY, pW);

    if (hasBias) {
        INDArray bias;
        if (kerasMajorVersion == 2 && weights.containsKey("bias"))
            bias = weights.get("bias");
        else if (kerasMajorVersion == 1 && weights.containsKey("b"))
            bias = weights.get("b");
        else
            throw new InvalidKerasConfigurationException(
                    "Keras SeparableConvolution2D layer does not contain bias parameter");
        this.weights.put(SeparableConvolutionParamInitializer.BIAS_KEY, bias);

    }

}
 

@Test
public void testPooling2D_Same() {
    int[] miniBatches = {1, 3, 5};
    int[] depths = {1, 3, 5};
    int[] inHeights = {5, 21};
    int[] inWidths = {5, 21};
    int[] strideH = {1, 2};
    int[] strideW = {1, 2};
    int[] sizeW = {1, 2, 3};
    int[] sizeH = {1, 2, 3};
    int[] padH = {0};
    int[] padW = {0};
    Pooling2D.Pooling2DType[] types = new Pooling2D.Pooling2DType[]{Pooling2D.Pooling2DType.AVG, Pooling2D.Pooling2DType.PNORM, Pooling2D.Pooling2DType.MAX,};

    int cnt = 0;

    for (Pooling2D.Pooling2DType type : types) {
        log.info("Trying pooling type: [{}]", type);
        for (int m : miniBatches) {
            for (int d : depths) {
                for (int h : inHeights) {
                    for (int w : inWidths) {
                        for (int sh : strideH) {
                            for (int sw : strideW) {
                                for (int kh : sizeH) {
                                    for (int kw : sizeW) {

                                        INDArray in = Nd4j.rand(new int[]{m, d, h, w});

                                        int[] outSize = getOutputSize(in, new int[]{kh, kw}, new int[]{sh, sw}, null, true);

                                        //Calculate padding for same mode:
                                        int pHTotal = (outSize[0]-1)*sh + kh - h;
                                        int pWTotal = (outSize[1]-1)*sw + kw - w;
                                        int padTop = pHTotal / 2;
                                        int padLeft = pWTotal / 2;

                                        INDArray col = Nd4j.create(new int[]{m, d, outSize[0], outSize[1], kh, kw}, 'c');
                                        INDArray col2 = col.permute(0, 1, 4, 5, 2, 3);
                                        //INDArray col = Nd4j.createUninitialized(new int[]{m, d, kh, kw, outSize[0], outSize[1]}, 'c');
                                        //INDArray col2 = col;

                                        Convolution.im2col(in, kh, kw, sh, sw, padTop, padLeft, true, col2);

                                        INDArray col2d = col.reshape('c', m * d * outSize[0] * outSize[1], kh * kw);

                                        INDArray output = Nd4j.create(m, d, outSize[0], outSize[1]);



                                        INDArray reduced = null;
                                        switch (type) {
                                            case PNORM:
                                                int pnorm = 3;

                                                Transforms.abs(col2d, false);
                                                Transforms.pow(col2d, pnorm, false);
                                                reduced = col2d.sum(1);
                                                Transforms.pow(reduced, (1.0 / pnorm), false);

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.PNORM, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        (double) pnorm, outSize[0], outSize[1], output);

                                                break;
                                            case MAX:
                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.MAX, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.max(1);
                                                break;
                                            case AVG:

                                                Convolution.pooling2D(in, kh, kw, sh, sw, padTop, padLeft, 1, 1,
                                                        true, Pooling2D.Pooling2DType.AVG, Pooling2D.Divisor.INCLUDE_PADDING,
                                                        0.0, outSize[0], outSize[1], output);

                                                reduced = col2d.mean(1);
                                                break;
                                        }

                                        reduced = reduced.reshape('c',m,d, outSize[0], outSize[1]);

                                        assertEquals("Failed opType: " + type, reduced, output);
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
 

public static INDArray activate(@NonNull INDArray boundingBoxPriors, @NonNull INDArray input, boolean nchw, LayerWorkspaceMgr layerWorkspaceMgr){
    if(!nchw)
        input = input.permute(0,3,1,2); //NHWC to NCHW

    long mb = input.size(0);
    long h = input.size(2);
    long w = input.size(3);
    long b = boundingBoxPriors.size(0);
    long c = input.size(1)/b-5;  //input.size(1) == b * (5 + C) -> C = (input.size(1)/b) - 5

    INDArray output = layerWorkspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), input.shape(), 'c');
    INDArray output5 = output.reshape('c', mb, b, 5+c, h, w);
    INDArray output4 = output;  //output.get(all(), interval(0,5*b), all(), all());
    INDArray input4 = input.dup('c');    //input.get(all(), interval(0,5*b), all(), all()).dup('c');
    INDArray input5 = input4.reshape('c', mb, b, 5+c, h, w);

    //X/Y center in grid: sigmoid
    INDArray predictedXYCenterGrid = input5.get(all(), all(), interval(0,2), all(), all());
    Transforms.sigmoid(predictedXYCenterGrid, false);

    //width/height: prior * exp(input)
    INDArray predictedWHPreExp = input5.get(all(), all(), interval(2,4), all(), all());
    INDArray predictedWH = Transforms.exp(predictedWHPreExp, false);
    Broadcast.mul(predictedWH, boundingBoxPriors.castTo(input.dataType()), predictedWH, 1, 2);  //Box priors: [b, 2]; predictedWH: [mb, b, 2, h, w]

    //Confidence - sigmoid
    INDArray predictedConf = input5.get(all(), all(), point(4), all(), all());   //Shape: [mb, B, H, W]
    Transforms.sigmoid(predictedConf, false);

    output4.assign(input4);

    //Softmax
    //TODO OPTIMIZE?
    INDArray inputClassesPreSoftmax = input5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    INDArray classPredictionsPreSoftmax2d = inputClassesPreSoftmax.permute(0,1,3,4,2) //[minibatch, b, c, h, w] To [mb, b, h, w, c]
            .dup('c').reshape('c', new long[]{mb*b*h*w, c});
    Transforms.softmax(classPredictionsPreSoftmax2d, false);
    INDArray postSoftmax5d = classPredictionsPreSoftmax2d.reshape('c', mb, b, h, w, c ).permute(0, 1, 4, 2, 3);

    INDArray outputClasses = output5.get(all(), all(), interval(5, 5+c), all(), all());   //Shape: [minibatch, C, H, W]
    outputClasses.assign(postSoftmax5d);

    if(!nchw)
        output = output.permute(0,2,3,1);       //NCHW to NHWC

    return output;
}
 

public static Pair<INDArray, String> getPermutedWithShape(char ordering, long rows, long cols, long seed) {
    Nd4j.getRandom().setSeed(seed);
    long len = rows * cols;
    INDArray arr = Nd4j.linspace(1, len, len).reshape(cols, rows);
    return new Pair<>(arr.permute(1, 0), "getPermutedWithShape(" + rows + "," + cols + "," + seed + ")");
}
 

@Override
public List<List<Writable>> next(int num) {
    List<File> files = new ArrayList<>(num);
    List<List<ImageObject>> objects = new ArrayList<>(num);

    for (int i = 0; i < num && hasNext(); i++) {
        File f = iter.next();
        this.currentFile = f;
        if (!f.isDirectory()) {
            files.add(f);
            objects.add(labelProvider.getImageObjectsForPath(f.getPath()));
        }
    }

    int nClasses = labels.size();

    INDArray outImg = Nd4j.create(files.size(), channels, height, width);
    INDArray outLabel = Nd4j.create(files.size(), 4 + nClasses, gridH, gridW);

    int exampleNum = 0;
    for (int i = 0; i < files.size(); i++) {
        File imageFile = files.get(i);
        this.currentFile = imageFile;
        try {
            this.invokeListeners(imageFile);
            Image image = this.imageLoader.asImageMatrix(imageFile);
            this.currentImage = image;
            Nd4j.getAffinityManager().ensureLocation(image.getImage(), AffinityManager.Location.DEVICE);

            outImg.put(new INDArrayIndex[]{point(exampleNum), all(), all(), all()}, image.getImage());

            List<ImageObject> objectsThisImg = objects.get(exampleNum);

            label(image, objectsThisImg, outLabel, exampleNum);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        exampleNum++;
    }

    if(!nchw) {
        outImg = outImg.permute(0, 2, 3, 1);        //NCHW to NHWC
        outLabel = outLabel.permute(0, 2, 3, 1);
    }
    return new NDArrayRecordBatch(Arrays.asList(outImg, outLabel));
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    //For RNN: need to sum up the score over each time step before returning.
    INDArray input = this.input;
    INDArray labels = this.labels;
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    if (layerConf().getRnnDataFormat() == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
        labels = input.permute(0, 2, 1);
    }
    INDArray input2d = TimeSeriesUtils.reshape3dTo2d(input, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = TimeSeriesUtils.reshape3dTo2d(labels, workspaceMgr, ArrayType.FF_WORKING_MEM);

    INDArray maskReshaped;
    if(this.maskArray != null){
        if(this.maskArray.rank() == 3){
            maskReshaped = TimeSeriesUtils.reshapePerOutputTimeSeriesMaskTo2d(this.maskArray, workspaceMgr, ArrayType.FF_WORKING_MEM);
        } else {
            maskReshaped = TimeSeriesUtils.reshapeTimeSeriesMaskToVector(this.maskArray, workspaceMgr, ArrayType.FF_WORKING_MEM);
        }
    } else {
        maskReshaped = null;
    }

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
            lossFunction.computeScoreArray(labels2d, input2d, layerConf().getActivationFn(), maskReshaped);
    //scoreArray: shape [minibatch*timeSeriesLength, 1]
    //Reshape it to [minibatch, timeSeriesLength] then sum over time step

    INDArray scoreArrayTs = TimeSeriesUtils.reshapeVectorToTimeSeriesMask(scoreArray, (int)input.size(0));
    INDArray summedScores = scoreArrayTs.sum(1);

    if (fullNetRegTerm != 0.0) {
        summedScores.addi(fullNetRegTerm);
    }

    return summedScores;
}