Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#rank()

protected INDArray revertReshape(INDArray toRevert, long minibatch){

        int axis = (rnnDataFormat == RNNFormat.NCW)? 2 : 1;
        if(axis < 0)
            axis += (toRevert.rank()+1);

        long[] newShape = new long[toRevert.rank()+1];
        newShape[0] = minibatch;
        newShape[1] = toRevert.size(0)/minibatch;
        for( int i=1; i<toRevert.rank(); i++ ){
            newShape[i+1] = toRevert.size(i);
        }

        INDArray reshaped = toRevert.reshape('c', newShape);

        int[] permute = ArrayUtil.invertPermutation(permuteAxes(toRevert.rank() + 1, axis));

        INDArray permuted = reshaped.permute(permute);
        return permuted;
    }
 

public static INDArray col2im(INDArray col, INDArray z, int sy, int sx, int ph, int pw, int h, int w, int dh, int dw ) {
    if (col.rank() != 6)
        throw new IllegalArgumentException("col2im input array must be rank 6");
    if (z.rank() != 4)
        throw new IllegalArgumentException("col2im output array must be rank 4");
    Col2Im col2Im = Col2Im.builder()
            .inputArrays(new  INDArray[]{col})
            .outputs(new INDArray[]{z})
            .conv2DConfig(Conv2DConfig.builder()
                    .sy(sy)
                    .sx(sx)
                    .dw(dw)
                    .dh(dh)
                    .kh(h)
                    .kw(w)
                    .ph(ph)
                    .pw(pw)
                    .build())
            .build();

    Nd4j.getExecutioner().exec(col2Im);

    return z;
}
 

private void validateData(INDArray label, INDArray labelMask) {
    if (label.rank() != 3) {
        throw new IllegalArgumentException(
                        "UnderSamplingByMaskingPreProcessor can only be applied to a time series dataset");
    }
    if (label.size(1) > 2) {
        throw new IllegalArgumentException(
                        "UnderSamplingByMaskingPreProcessor can only be applied to labels that represent binary classes. Label size was found to be "
                                        + label.size(1) + ".Expecting size=1 or size=2.");
    }
    if (label.size(1) == 2) {
        //check if label is of size one hot
        if (!label.sum(1).mul(labelMask).equals(labelMask)) {
            throw new IllegalArgumentException("Labels of size minibatchx2xtimesteps are expected to be one hot."
                            + label.toString() + "\n is not one-hot");
        }
    }
}
 

/**
 * Check if the given indexes
 * over the specified array
 * are searching for a scalar
 * @param indexOver the array to index over
 * @param indexes the index query
 * @return true if the given indexes are searching
 * for a scalar false otherwise
 */
public static boolean isScalar(INDArray indexOver, INDArrayIndex... indexes) {
    boolean allOneLength = true;
    for (int i = 0; i < indexes.length; i++) {
        allOneLength = allOneLength && indexes[i].length() == 1;
    }

    int numNewAxes = NDArrayIndex.numNewAxis(indexes);
    if (allOneLength && numNewAxes == 0 && indexes.length == indexOver.rank())
        return true;
    else if (allOneLength && indexes.length == indexOver.rank() - numNewAxes) {
        return allOneLength;
    }

    return allOneLength;
}
 

/**
 * Normalize a data array
 *
 * @param array the data to normalize
 * @param stats statistics of the data population
 */
@Override
public void preProcess(INDArray array, INDArray maskArray, DistributionStats stats) {
    if (array.rank() <= 2) {
        array.subiRowVector(stats.getMean().castTo(array.dataType()));
        array.diviRowVector(filteredStd(stats).castTo(array.dataType()));
    }
    // if array Rank is 3 (time series) samplesxfeaturesxtimesteps
    // if array Rank is 4 (images) samplesxchannelsxrowsxcols
    // both cases operations should be carried out in dimension 1
    else {
        Nd4j.getExecutioner().execAndReturn(new BroadcastSubOp(array, stats.getMean().castTo(array.dataType()), array, 1));
        Nd4j.getExecutioner().execAndReturn(new BroadcastDivOp(array, filteredStd(stats).castTo(array.dataType()), array, 1));
    }

    if (maskArray != null) {
        DataSetUtil.setMaskedValuesToZero(array, maskArray);
    }
}
 

public static RealMatrix convertToApacheMatrix(INDArray matrix) {
    if (matrix.rank() != 2)
        throw new IllegalArgumentException("Input rank is not 2 (not matrix)");
    long[] shape = matrix.shape();

    if (matrix.columns() > Integer.MAX_VALUE || matrix.rows() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    BlockRealMatrix out = new BlockRealMatrix((int) shape[0], (int) shape[1]);
    for (int i = 0; i < shape[0]; i++) {
        for (int j = 0; j < shape[1]; j++) {
            double value = matrix.getDouble(i, j);
            out.setEntry(i, j, value);
        }
    }
    return out;
}
 

public static INDArray col2im(INDArray col, INDArray z, int sH, int sW, int pH, int pW, int kH, int kW,
                              int dH, int dW) {
    if (col.rank() != 6)
        throw new IllegalArgumentException("col2im input array must be rank 6");
    if (z.rank() != 4)
        throw new IllegalArgumentException("col2im output array must be rank 4");
    Col2Im col2Im = Col2Im.builder()
            .inputArrays(new INDArray[]{col})
            .outputs(new INDArray[]{z})
            .conv2DConfig(Conv2DConfig.builder()
                    .sH(sH)
                    .sW(sW)
                    .dH(dH)
                    .dW(dW)
                    .kH(kH)
                    .kW(kW)
                    .pH(pH)
                    .pW(pW)
                    .build())
            .build();

    Nd4j.getExecutioner().execAndReturn(col2Im);

    return z;
}
 

@Override
public NDArrayAnalysisCounter add(Writable writable) {
    NDArrayWritable n = (NDArrayWritable) writable;
    INDArray arr = n.get();
    countTotal++;
    if (arr == null) {
        countNull++;
    } else {
        minLength = Math.min(minLength, arr.length());
        maxLength = Math.max(maxLength, arr.length());

        int r = arr.rank();
        if (countsByRank.containsKey(arr.rank())) {
            countsByRank.put(r, countsByRank.get(r) + 1);
        } else {
            countsByRank.put(r, 1L);
        }

        totalNDArrayValues += arr.length();
        minValue = Math.min(minValue, arr.minNumber().doubleValue());
        maxValue = Math.max(maxValue, arr.maxNumber().doubleValue());
    }

    return this;
}
 

protected INDArray adjustRank(INDArray words) {
    if (lookupTable instanceof InMemoryLookupTable) {
        InMemoryLookupTable l = (InMemoryLookupTable) lookupTable;

        INDArray syn0 = l.getSyn0();
        if (!words.dataType().equals(syn0.dataType())) {
            return words.castTo(syn0.dataType());
        }
        if (words.rank() == 0 || words.rank() > 2) {
            throw new IllegalStateException("Invalid rank for wordsNearest method");
        } else if (words.rank() == 1) {
            return words.reshape(1, -1);
        }
    }
    return words;
}
 

/**
 * Reshape time series mask arrays. This should match the assumptions (f order, etc) in RnnOutputLayer
 * This reshapes from [X,1] to [X,1,1,1] suitable for per-example masking in CNNs
 * @param timeSeriesMask    Mask array to reshape for CNN per-example masking
 * @return                  Mask array as 4D CNN mask array: [X, 1, 1, 1]
 */
public static INDArray reshapeTimeSeriesMaskToCnn4dMask(INDArray timeSeriesMask, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType) {
    if (timeSeriesMask.rank() != 2)
        throw new IllegalArgumentException("Cannot reshape mask: rank is not 2");

    if (timeSeriesMask.ordering() != 'f' || !Shape.hasDefaultStridesForShape(timeSeriesMask))
        timeSeriesMask = workspaceMgr.dup(arrayType, timeSeriesMask, 'f');

    return workspaceMgr.leverageTo(arrayType, timeSeriesMask.reshape('f', timeSeriesMask.length(), 1, 1, 1));
}
 

/**
 * Create the dimensions for the flatbuffer builder
 * @param bufferBuilder the buffer builder to use
 * @param arr the input array
 * @return
 */
public static int createDims(FlatBufferBuilder bufferBuilder,INDArray arr) {
    int[] tensorDimOffsets = new int[arr.rank()];
    int[] nameOffset = new int[arr.rank()];
    for(int i = 0; i < tensorDimOffsets.length; i++) {
        nameOffset[i] = bufferBuilder.createString("");
        tensorDimOffsets[i] = TensorDim.createTensorDim(bufferBuilder,arr.size(i),nameOffset[i]);
    }

    return Tensor.createShapeVector(bufferBuilder,tensorDimOffsets);
}
 

/**
 * Reverses whole array for compatibility with OldReverse.
 *
 * Note that otherwise, passing null or empty dimensions will result in a noop.
 */
public Reverse(INDArray x, INDArray z){
    super(new INDArray[]{x}, new INDArray[]{z});
    this.dimensions = new int[x.rank()];
    for(int i = 0 ; i < this.dimensions.length ; i++)
        this.dimensions[i] = i;
    addIArgument(dimensions);
}
 

public static INDArray tailor2d(@NonNull INDArray data, INDArray mask) {
    switch (data.rank()) {
        case 1:
        case 2:
            return data;
        case 3:
            return tailor3d2d(data, mask);
        case 4:
            return tailor4d2d(data);
        default:
            throw new RuntimeException("Unsupported data rank");
    }
}
 

private static INDArray getExample(int idx, INDArray from){
    INDArrayIndex[] idxs = new INDArrayIndex[from.rank()];
    idxs[0] = NDArrayIndex.interval(idx, idx, true);    //Use interval to avoid collapsing point dimension
    for( int i=1; i<from.rank(); i++){
        idxs[i] = NDArrayIndex.all();
    }
    return from.get(idxs);
}
 

/**
 * Create the dimensions for the flatbuffer builder
 * @param bufferBuilder the buffer builder to use
 * @param arr the input array
 * @return
 */
public static int createDims(FlatBufferBuilder bufferBuilder,INDArray arr) {
    int[] tensorDimOffsets = new int[arr.rank()];
    int[] nameOffset = new int[arr.rank()];
    for(int i = 0; i < tensorDimOffsets.length; i++) {
        nameOffset[i] = bufferBuilder.createString("");
        tensorDimOffsets[i] = TensorDim.createTensorDim(bufferBuilder,arr.size(i),nameOffset[i]);
    }

    return Tensor.createShapeVector(bufferBuilder,tensorDimOffsets);
}
 

public Nd4jVector(INDArray vector) {
    if (vector.rank() != 1) {
        new IllegalArgumentException();
    }
    this.size = (int) vector.length();
    this.order = vector.ordering();
    this.vector = vector;
}
 

public Nd4jMatrix(INDArray matrix) {
    if (matrix.rank() != 2) {
        new IllegalArgumentException();
    }
    this.rowSize = matrix.rows();
    this.columnSize = matrix.columns();
    this.size = rowSize * columnSize;
    this.order = matrix.ordering();
    this.matrix = matrix;
}
 

public static void setMaskedValuesToZero(INDArray data, INDArray mask) {
    if (mask == null || data.rank() != 3)
        return;

    Nd4j.getExecutioner().exec(new BroadcastMulOp(data, mask, data, 0, 2));
}
 

@Override
    public INDArray operate(@Nonnull final INDArray W_tl)
            throws DimensionMismatchException {
        if (W_tl.rank() != 2 || W_tl.shape()[0] != numTargets || W_tl.shape()[1] != numLatents) {
            throw new DimensionMismatchException(W_tl.length(), numTargets * numLatents);
        }
        long startTimeRFFT = System.nanoTime();
        /* forward rfft */
        final INDArray W_kl = Nd4j.create(fftSize, numLatents);
        IntStream.range(0, numLatents).parallel().forEach(li ->
                W_kl.get(NDArrayIndex.all(), NDArrayIndex.point(li)).assign(
                        Nd4j.create(F_tt.getForwardFFT(W_tl.get(NDArrayIndex.all(), NDArrayIndex.point(li))),
                                new int[]{fftSize, 1})));
        long endTimeRFFT = System.nanoTime();

        /* apply the preconditioner in the Fourier space */
        long startTimePrecond = System.nanoTime();
        final Map<LinearlySpacedIndexBlock, INDArray> W_kl_map = CoverageModelSparkUtils.partitionINDArrayToMap(fourierSpaceBlocks, W_kl);
        final Broadcast<Map<LinearlySpacedIndexBlock, INDArray>> W_kl_bc = ctx.broadcast(W_kl_map);
        final JavaPairRDD<LinearlySpacedIndexBlock, INDArray> preconditionedWRDD = linOpPairRDD
                .mapToPair(p -> {
                    final INDArray W_kl_chuck = W_kl_bc.value().get(p._1);
                    final INDArray linOp_chunk = p._2;
                    final int blockSize = linOp_chunk.shape()[0];
                    final List<INDArray> linOpWList = IntStream.range(0, blockSize).parallel()
                            .mapToObj(k -> CoverageModelEMWorkspaceMathUtils.linsolve(linOp_chunk.get(NDArrayIndex.point(k)),
                                    W_kl_chuck.get(NDArrayIndex.point(k))))
                            .collect(Collectors.toList());
                    return new Tuple2<>(p._1, Nd4j.vstack(linOpWList));
                });
        W_kl.assign(CoverageModelSparkUtils.assembleINDArrayBlocksFromRDD(preconditionedWRDD, 0));
        W_kl_bc.destroy();
//        final JavaPairRDD<LinearlySpacedIndexBlock, INDArray> W_kl_RDD = CoverageModelSparkUtils.rddFromINDArray(W_kl,
//                fourierSpaceBlocks, ctx, true);
//        W_kl.assign(CoverageModelSparkUtils.assembleINDArrayBlocks(linOpPairRDD.join((W_kl_RDD))
//                .mapValues(p -> {
//                    final INDArray linOp = p._1;
//                    final INDArray W = p._2;
//                    final int blockSize = linOp.shape()[0];
//                    final List<INDArray> linOpWList = IntStream.range(0, blockSize).parallel().mapToObj(k ->
//                            CoverageModelEMWorkspaceMathUtils.linsolve(linOp.get(NDArrayIndex.point(k)),
//                                    W.get(NDArrayIndex.point(k))))
//                            .collect(Collectors.toList());
//                    return Nd4j.vstack(linOpWList);
//                }), false));
//        W_kl_RDD.unpersist();
        long endTimePrecond = System.nanoTime();

        /* irfft */
        long startTimeIRFFT = System.nanoTime();
        final INDArray res = Nd4j.create(numTargets, numLatents);
        IntStream.range(0, numLatents).parallel().forEach(li ->
                res.get(NDArrayIndex.all(), NDArrayIndex.point(li)).assign(
                        F_tt.getInverseFFT(W_kl.get(NDArrayIndex.all(), NDArrayIndex.point(li)))));
        long endTimeIRFFT = System.nanoTime();

        logger.debug("Local FFT timing: " + (endTimeRFFT - startTimeRFFT + endTimeIRFFT - startTimeIRFFT)/1000000 + " ms");
        logger.debug("Spark preconditioner application timing: " + (endTimePrecond - startTimePrecond)/1000000 + " ms");

        return res;
    }
 

/**
 * The number of vectors
 * in each slice of an ndarray.
 * @param arr the array to
 *            get the number
 *            of vectors per slice for
 * @return the number of vectors per slice
 */
public static long vectorsPerSlice(INDArray arr) {
    if (arr.rank() > 2) {
        return ArrayUtil.prodLong(new long[] {arr.size(-1), arr.size(-2)});
    }

    return arr.slices();
}