org.nd4j.linalg.util.ArrayUtil Java Examples

private void addArgs() {
    addIArgument(new long[]{
            config.getDT(),
            config.getDW(),
            config.getDH(),
            config.getPT(),
            config.getPW(),
            config.getPH(),
            config.getDilationT(),
            config.getDilationW(),
            config.getDilationH(),
            config.getAT(),
            config.getAW(),
            config.getAH(),
            ArrayUtil.fromBoolean(config.isBiasUsed())});


}
 

public INDArray mapTensorProto(OnnxProto3.TensorProto tensor) {
    if(tensor == null)
        return null;


    DataBuffer.Type type = nd4jTypeFromOnnxType(tensor.getDataType());

    ByteString bytes = tensor.getRawData();
    ByteBuffer byteBuffer = bytes.asReadOnlyByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer directAlloc = ByteBuffer.allocateDirect(byteBuffer.capacity()).order(ByteOrder.nativeOrder());
    directAlloc.put(byteBuffer);
    directAlloc.rewind();
    long[] shape = getShapeFromTensor(tensor);
    DataBuffer buffer = Nd4j.createBuffer(directAlloc,type, ArrayUtil.prod(shape));
    INDArray arr = Nd4j.create(buffer).reshape(shape);
    return arr;
}
 

/**
 * Compute the offset given an array of offsets.
 * The offset is computed(for both fortran an d c ordering) as:
 * sum from i to n - 1 o[i] * s[i]
 * where i is the index o is the offset and s is the stride
 * Notice the -1 at the end.
 * @param strides the strides to compute the offset for
 * @param offsets the offsets for each dimension
 * @return the offset that should be used for indexing
 */
public static long offset(long[] strides, long[] offsets) {
    int ret = 0;

    if (ArrayUtil.prod(offsets) == 1) {
        for (int i = 0; i < offsets.length; i++) {
            ret += offsets[i] * strides[i];
        }
    } else {
        for (int i = 0; i < offsets.length; i++) {
            ret += offsets[i] * strides[i];
        }

    }

    return ret;
}
 

/**
 * Create an ndarray from the specified slices
 * and the given shape
 *
 * @param slices   the slices of the ndarray
 * @param shape    the final shape of the ndarray
 * @param stride   the stride of the ndarray
 * @param ordering the ordering for the ndarray
 */
public BaseComplexNDArray(List<IComplexNDArray> slices, int[] shape, int[] stride, char ordering) {
    this(new float[ArrayUtil.prod(shape) * 2]);
    List<IComplexNumber> list = new ArrayList<>();
    for (int i = 0; i < slices.size(); i++) {
        IComplexNDArray flattened = slices.get(i).ravel();
        for (int j = 0; j < flattened.length(); j++)
            list.add(flattened.getComplex(j));
    }
    throw new UnsupportedOperationException();
    /*
    
    this.ordering = ordering;
    this.data = Nd4j.createBuffer(ArrayUtil.prod(shape) * 2);
    this.stride = stride;
    init(shape);
    
    int count = 0;
    for (int i = 0; i < list.size(); i++) {
        putScalar(count, list.get(i));
        count++;
    }*/
}
 

@Override
public List<long[]> calculateOutputShape() {
    if(mMulTranspose == null)
        mMulTranspose = MMulTranspose.allFalse();
    List<long[]> ret = new ArrayList<>(1);
    long[] aShape = mMulTranspose.isTransposeA() ? ArrayUtil.reverseCopy(larg().getShape()) : larg().getShape();
    long[] bShape = mMulTranspose.isTransposeB() ? ArrayUtil.reverseCopy(rarg().getShape()) : rarg().getShape();
    if(Shape.isPlaceholderShape(aShape) || Shape.isPlaceholderShape(bShape))
        return Collections.emptyList();

    if(aShape != null && bShape != null) {
        val shape =  Shape.getMatrixMultiplyShape(aShape,bShape);
        ret.add(shape);
    }
    if(!ret.isEmpty()) {
        for(int i = 0; i < ret.get(0).length; i++) {
            if(ret.get(0)[i] < 1)
                throw new ND4JIllegalStateException("Invalid shape computed at index " +  i);
        }
    }
    return ret;
}
 

/**
 * There's SMALL chance this test will randomly fail, since spread isn't too big
 * @throws Exception
 */
@Test
public void testHalfVectors1() throws Exception {
    int[] array1 = ArrayUtil.buildHalfVector(new Random(12), 20);
    int[] array2 = ArrayUtil.buildHalfVector(new Random(75), 20);

    assertFalse(Arrays.equals(array1, array2));

    assertEquals(20, array1.length);
    assertEquals(20, array2.length);

    for (int i = 0; i < array1.length; i++) {
        if (i >= array1.length / 2) {
            assertEquals("Failed on element [" + i + "]", -1, array1[i]);
            assertEquals("Failed on element [" + i + "]", -1, array2[i]);
        } else {
            assertNotEquals("Failed on element [" + i + "]", -1, array1[i]);
            assertNotEquals("Failed on element [" + i + "]", -1, array2[i]);
        }
    }
}
 

public static long offset(int[] strides, long[] offsets) {
    int ret = 0;

    if (ArrayUtil.prodLong(offsets) == 1) {
        for (int i = 0; i < offsets.length; i++) {
            ret += offsets[i] * strides[i];
        }
    } else {
        for (int i = 0; i < offsets.length; i++) {
            ret += offsets[i] * strides[i];
        }

    }

    return ret;
}
 

public BaseTransformOp(SameDiff sameDiff,
                       SDVariable i_v,
                       int[] shape,
                       boolean inPlace,
                       Object[] extraArgs) {
    // FIXME: int cast !
    this(sameDiff, i_v, ArrayUtil.toLongArray(shape), inPlace, extraArgs);
}
 

@Test
@Ignore
public void testArrayRemoveIndex5() throws Exception {
    //INDArray arraySource = Nd4j.create(new float[]{1,2,3,4,5,6,7,8});
    int[] arraySource = new int[] {1,2,3,4,5,6,7,8};

    int[] dst = ArrayUtil.removeIndex(arraySource, new int[]{Integer.MAX_VALUE});

    assertEquals(8, dst.length);
    assertEquals(1, dst[0]);
    assertEquals(8, dst[7]);
}
 

/**
 *
 * Choose tensor dimension for operations with one argument: x=Op(x) or similar<br>
 * When doing some operations in parallel, it is necessary to break up
 * operations along a dimension to
 * give a set of 1d tensors. The dimension that this is done on is important for performance reasons;
 * in summary we want to both minimize the number of tensors
 * , but also minimize the separation between
 * elements in the buffer (so the resulting operation is efficient - i.e., avoids cache thrashing).
 * However, achieving both minimal number
 * of tensors and are not always possible.
 * @param x NDArray that we want to split
 * @return The best dimension to split on
 */
public static int chooseElementWiseTensorDimension(INDArray x) {
    if (x.isVector())
        return ArrayUtil.argMax(x.shape()); //Execute along the vector

    //doing argMin(max(x.stride(i),y.stride(i))) minimizes the maximum
    //separation between elements (helps CPU cache) BUT might result in a huge number
    //of tiny ops - i.e., addi on NDArrays with shape [5,10^6]
    int opAlongDimensionMinStride = ArrayUtil.argMin(x.stride());

    //doing argMax on shape gives us smallest number of largest tensors
    //but may not be optimal in terms of element separation (for CPU cache etc)
    int opAlongDimensionMaxLength = ArrayUtil.argMax(x.shape());

    //Edge cases: shapes with 1s in them can have stride of 1 on the dimensions of length 1
    if (x.isVector() || x.size(opAlongDimensionMinStride) == 1)
        return opAlongDimensionMaxLength;

    //Using a heuristic approach here: basically if we get >= 10x as many tensors using the minimum stride
    //dimension vs. the maximum size dimension, use the maximum size dimension instead
    //The idea is to avoid choosing wrong dimension in cases like shape=[10,10^6]
    //Might be able to do better than this with some additional thought
    int nOpsAlongMinStride = ArrayUtil.prod(ArrayUtil.removeIndex(x.shape(), opAlongDimensionMinStride));
    int nOpsAlongMaxLength = ArrayUtil.prod(ArrayUtil.removeIndex(x.shape(), opAlongDimensionMaxLength));
    if (nOpsAlongMinStride <= 10 * nOpsAlongMaxLength)
        return opAlongDimensionMinStride;
    else
        return opAlongDimensionMaxLength;
}
 

@Test
public void testRowVectorShapeOneOneOffset() {
    INDArray arr = Nd4j.create(2, 2);
    ShapeOffsetResolution resolution = new ShapeOffsetResolution(arr);
    //row 0
    resolution.exec(NDArrayIndex.point(1));
    long[] oneIndexShape = ArrayUtil.copy(resolution.getShapes());
    assertArrayEquals(new long[] {1, 2}, oneIndexShape);
    assertEquals(2, resolution.getOffset());
    long[] oneIndexStrides = ArrayUtil.copy(resolution.getStrides());
    assertArrayEquals(new long[] {1, 1}, oneIndexStrides);

}
 

@Override
public INDArray subArray(ShapeOffsetResolution resolution) {
    long[] offsets = resolution.getOffsets();
    int[] shape = LongUtils.toInts(resolution.getShapes());
    int[] stride = LongUtils.toInts(resolution.getStrides());
    int[] flags = resolution.getFixed();
    flags = updateFlags(flags, shape);
    long offset = (int) (offset() + resolution.getOffset());
    int newRank = shape.length;
    long[] sparseOffsets = createSparseOffsets(offset);
    int[] newAxis = createHiddenDimensions(resolution.getPrependAxis());


    if (offset() + resolution.getOffset() >= Integer.MAX_VALUE)
        throw new IllegalArgumentException("Offset of array can not be >= Integer.MAX_VALUE");

    if (offsets.length != newRank)
        throw new IllegalArgumentException("Invalid offset " + Arrays.toString(offsets));
    if (stride.length != newRank)
        throw new IllegalArgumentException("Invalid stride " + Arrays.toString(stride));

    if (shape.length == rank() && Shape.contentEquals(shape, shapeOf())) {
        if (ArrayUtil.isZero(offsets)) {
            return this;
        } else {
            throw new IllegalArgumentException("Invalid subArray offsets");
        }
    }
    DataBuffer newSparseInformation = Nd4j.getSparseInfoProvider().createSparseInformation(flags, sparseOffsets,
                    newAxis, underlyingRank());
    return create(values, indices, newSparseInformation, Arrays.copyOf(shape, shape.length));

}
 

@Override
public long n() {
    if(n == 0) {
        if(arg() != null)
            this.n = ArrayUtil.prod(arg().getShape());

    }
    return n;
}
 

/** Can we do the transform op (X = Op(X,Y)) directly on the arrays without breaking them up into 1d tensors first? */
public static boolean canDoOpDirectly(INDArray x, INDArray y) {
    if (x.isVector())
        return true;
    if (x.ordering() != y.ordering())
        return false; //other than vectors, elements in f vs. c NDArrays will never line up
    if (x.elementWiseStride() < 1 || y.elementWiseStride() < 1)
        return false;
    //Full buffer + matching strides -> implies all elements are contiguous (and match)
    //Need strides to match, otherwise elements in buffer won't line up (i.e., c vs. f order arrays)
    long l1 = x.lengthLong();
    long dl1 = x.data().length();
    long l2 = y.lengthLong();
    long dl2 = y.data().length();
    long[] strides1 = x.stride();
    long[] strides2 = y.stride();
    boolean equalStrides = Arrays.equals(strides1, strides2);
    if (l1 == dl1 && l2 == dl2 && equalStrides)
        return true;

    //Strides match + are same as a zero offset NDArray -> all elements are contiguous (and match)
    if (equalStrides) {
        long[] shape1 = x.shape();
        long[] stridesAsInit = (x.ordering() == 'c' ? ArrayUtil.calcStrides(shape1)
                        : ArrayUtil.calcStridesFortran(shape1));
        boolean stridesSameAsInit = Arrays.equals(strides1, stridesAsInit);
        return stridesSameAsInit;
    }

    return false;
}
 

/**
 * Create a complex ndarray with the given complex doubles.
 * Note that this maybe an easier setup than the new float
 *
 * @param newData the new data for this array
 * @param shape   the shape of the ndarray
 */
public BaseComplexNDArray(IComplexNumber[] newData, int[] shape) {
    super(new float[ArrayUtil.prod(shape) * 2]);
    /*  init(shape);
    for (int i = 0; i < length; i++)
        putScalar(i, newData[i].asDouble());
    */
    throw new UnsupportedOperationException();

}
 

protected void buildZ(IndexAccumulation op, int... dimension) {
    Arrays.sort(dimension);

    for (int i = 0; i < dimension.length; i++) {
        if (dimension[i] < 0)
            dimension[i] += op.x().rank();
    }

    //do op along all dimensions
    if (dimension.length == op.x().rank())
        dimension = new int[] {Integer.MAX_VALUE};


    long[] retShape = Shape.wholeArrayDimension(dimension) ? new long[] {1, 1}
            : ArrayUtil.removeIndex(op.x().shape(), dimension);
    //ensure vector is proper shape
    if (retShape.length == 1) {
        if (dimension[0] == 0)
            retShape = new long[] {1, retShape[0]};
        else
            retShape = new long[] {retShape[0], 1};
    } else if (retShape.length == 0) {
        retShape = new long[] {1, 1};
    }

    if(op.z() == null || op.z() == op.x()){
        INDArray ret = null;
        if (Math.abs(op.zeroDouble()) < Nd4j.EPS_THRESHOLD) {
            ret = Nd4j.zeros(retShape);
        } else {
            ret = Nd4j.valueArrayOf(retShape, op.zeroDouble());
        }

        op.setZ(ret);
    } else if(!Arrays.equals(retShape, op.z().shape())){
        throw new IllegalStateException("Z array shape does not match expected return type for op " + op
                + ": expected shape " + Arrays.toString(retShape) + ", z.shape()=" + Arrays.toString(op.z().shape()));
    }
}
 

@Override
public void resolvePropertiesFromSameDiffBeforeExecution() {
    super.resolvePropertiesFromSameDiffBeforeExecution();
    if (broadcast != null && numInputArguments() < 2) {
        if (numInputArguments() == 0) {
            addInputArgument(args()[0].getArr(), Nd4j.create(ArrayUtil.toFloats(broadcast)).reshape(broadcast.length));

        } else if (numInputArguments() == 1) {
            addInputArgument(Nd4j.create(ArrayUtil.toFloats(broadcast)));
        }

    }

    if (numIArguments() < 1) {
        addIArgument(axis);
    }

    if (numOutputArguments() < getDescriptor().getNumOutputs()) {
        val outputs = outputVariables();
        for (int i = 0; i < outputs.length; i++) {
            val output = outputs[i].getArr();
            addOutputArgument(output);
        }
    }


}
 

protected void addArgs() {
    addIArgument(config.getKh(),
            config.getKw(),
            config.getSy(),
            config.getSx(),
            config.getPh(),
            config.getPw(),
            config.getDh(),
            config.getDw(),
            ArrayUtil.fromBoolean(config.isSameMode()),
            ArrayUtil.fromBoolean(config.isNHWC()));
}
 

@Override
public INDArray nextGaussian(char order, long[] shape) {
    long length = ArrayUtil.prodLong(shape);
    INDArray ret = Nd4j.create(shape, order);

    DataBuffer data = ret.data();
    for (long i = 0; i < length; i++) {
        data.put(i, nextGaussian());
    }

    return ret;
}
 

public static boolean hasDefaultStridesForShape(INDArray input){
    if(!strideDescendingCAscendingF(input)){
        return false;
    }
    char order = input.ordering();
    long[] defaultStrides;
    if(order == 'f'){
        defaultStrides = ArrayUtil.calcStridesFortran(input.shape());
    } else {
        defaultStrides = ArrayUtil.calcStrides(input.shape());
    }
    return Arrays.equals(input.stride(), defaultStrides);
}
 

/**
 * Iterate over a pair of coordinates
 * @param dimension
 * @param n
 * @param size
 */
public static void iterate(int dimension, int n, int[] size, int[] res, CoordinateFunction func) {
    if (dimension >= n) { //stop clause
        func.process(ArrayUtil.toLongArray(res));
        return;
    }
    for (int i = 0; i < size[dimension]; i++) {
        res[dimension] = i;
        iterate(dimension + 1, n, ArrayUtil.toLongArray(size), ArrayUtil.toLongArray(res), func);
    }
}
 

@Test
public void testArrayRemoveIndex4() throws Exception {
    //INDArray arraySource = Nd4j.create(new float[]{1,2,3,4,5,6,7,8});
    int[] arraySource = new int[] {1,2,3,4,5,6,7,8};

    int[] dst = ArrayUtil.removeIndex(arraySource, new int[]{0});

    assertEquals(7, dst.length);
    assertEquals(2, dst[0]);
    assertEquals(8, dst[6]);
}
 

@Override
public INDArray getNDArrayFromTensor(String tensorName, OnnxProto3.TypeProto.Tensor tensorProto, OnnxProto3.GraphProto graph) {
    DataBuffer.Type type = dataTypeForTensor(tensorProto);
    if(!tensorProto.isInitialized()) {
        throw new ND4JIllegalStateException("Unable to retrieve ndarray. Tensor was not initialized");
    }

    OnnxProto3.TensorProto tensor = null;
    for(int i = 0; i < graph.getInitializerCount(); i++) {
        val initializer = graph.getInitializer(i);
        if(initializer.getName().equals(tensorName)) {
            tensor = initializer;
            break;
        }
    }

    if(tensor == null)
        return null;

    ByteString bytes = tensor.getRawData();
    ByteBuffer byteBuffer = bytes.asReadOnlyByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer directAlloc = ByteBuffer.allocateDirect(byteBuffer.capacity()).order(ByteOrder.nativeOrder());
    directAlloc.put(byteBuffer);
    directAlloc.rewind();
    long[] shape = getShapeFromTensor(tensorProto);
    DataBuffer buffer = Nd4j.createBuffer(directAlloc,type, ArrayUtil.prod(shape));
    INDArray arr = Nd4j.create(buffer).reshape(shape);
    return arr;
}
 

/**
 *
 * @param data
 * @param ordering
 */
public BaseNDArray(double[][] data, char ordering) {
    this(internalCreateBuffer(ordering == 'c' ? ArrayUtil.flatten(data) : ArrayUtil.flattenF(data)),
            new int[] {data.length, data[0].length},
            Nd4j.getStrides(new int[] {data.length, data[0].length}, ordering), 0, ordering);

    for (int r = 0; r < rows(); r++) {
        assert (data[r].length == columns());
    }
}
 

/**
 *
 * @param newShape the new shape possibly
 *                 containing a negative number
 * @param shape the shape to calculate from
 * @return
 */
public static int[] resolveNegativeShapeIfNeccessary(int[] newShape,int[] shape) {
    int numberNegativesOnes = 0;
    for (int i = 0; i < shape.length; i++) {
        if (shape[i] < 0) {
            if (numberNegativesOnes >= 1)
                throw new IllegalArgumentException("Only one dimension can be negative ones");

            numberNegativesOnes++;

            int shapeLength = 1;
            for (int j = 0; j < shape.length; j++)
                if (shape[j] >= 1)
                    shapeLength *= shape[j];
            int realShape = Math.abs(ArrayUtil.prod(newShape) / shapeLength);
            int[] thisNewShape = new int[shape.length];
            for (int j = 0; j < shape.length; j++) {
                if (i != j) {
                    thisNewShape[j] = shape[j];
                } else
                    thisNewShape[j] = realShape;
            }

            shape = thisNewShape;
            break;

        }

    }

    for(int i = 0; i < shape.length; i++) {
        if(shape[i] == 0) {
            shape[i] = 1;
        }
    }

    return shape;

}
 

public static List<Pair<INDArray, String>> get3dSubArraysWithShape(long seed, long... shape) {
    List<Pair<INDArray, String>> list = new ArrayList<>();
    String baseMsg = "get3dSubArraysWithShape(" + seed + "," + Arrays.toString(shape) + ")";
    //Create and return various sub arrays:
    Nd4j.getRandom().setSeed(seed);
    val newShape1 = Arrays.copyOf(shape, shape.length);
    newShape1[0] += 5;
    int len = ArrayUtil.prod(newShape1);
    INDArray temp1 = Nd4j.linspace(1, len, len).reshape(newShape1);
    INDArray subset1 = temp1.get(NDArrayIndex.interval(2, shape[0] + 2), NDArrayIndex.all(), NDArrayIndex.all());
    list.add(new Pair<>(subset1, baseMsg + ".get(0)"));

    val newShape2 = Arrays.copyOf(shape, shape.length);
    newShape2[1] += 5;
    int len2 = ArrayUtil.prod(newShape2);
    INDArray temp2 = Nd4j.linspace(1, len2, len2).reshape(newShape2);
    INDArray subset2 = temp2.get(NDArrayIndex.all(), NDArrayIndex.interval(3, shape[1] + 3), NDArrayIndex.all());
    list.add(new Pair<>(subset2, baseMsg + ".get(1)"));

    val newShape3 = Arrays.copyOf(shape, shape.length);
    newShape3[2] += 5;
    int len3 = ArrayUtil.prod(newShape3);
    INDArray temp3 = Nd4j.linspace(1, len3, len3).reshape(newShape3);
    INDArray subset3 = temp3.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(4, shape[2] + 4));
    list.add(new Pair<>(subset3, baseMsg + ".get(2)"));

    val newShape4 = Arrays.copyOf(shape, shape.length);
    newShape4[0] += 5;
    newShape4[1] += 5;
    newShape4[2] += 5;
    int len4 = ArrayUtil.prod(newShape4);
    INDArray temp4 = Nd4j.linspace(1, len4, len4).reshape(newShape4);
    INDArray subset4 = temp4.get(NDArrayIndex.interval(4, shape[0] + 4), NDArrayIndex.interval(3, shape[1] + 3),
                    NDArrayIndex.interval(2, shape[2] + 2));
    list.add(new Pair<>(subset4, baseMsg + ".get(3)"));

    return list;
}
 

@Test
public void testRowVectorShapeTwoOneOffset() {
    INDArray arr = Nd4j.create(2, 2);
    ShapeOffsetResolution resolution = new ShapeOffsetResolution(arr);
    //row 0
    resolution.exec(NDArrayIndex.point(1), NDArrayIndex.all());
    long[] oneIndexShape = ArrayUtil.copy(resolution.getShapes());
    assertArrayEquals(new long[] {1, 2}, oneIndexShape);
    long[] oneIndexOffsets = ArrayUtil.copy(resolution.getOffsets());
    assertArrayEquals(new long[] {0, 0}, oneIndexOffsets);
    assertEquals(2, resolution.getOffset());
    long[] oneIndexStrides = ArrayUtil.copy(resolution.getStrides());
    assertArrayEquals(new long[] {1, 1}, oneIndexStrides);

}
 

public static List<Pair<INDArray, String>> get5dReshapedWithShape(int seed, int... shape) {
    Nd4j.getRandom().setSeed(seed);
    int[] shape2d = {shape[0] * shape[2], shape[4], shape[1] * shape[3]};
    INDArray array3d = Nd4j.rand(shape2d);
    INDArray array5d = array3d.reshape(ArrayUtil.toLongArray(shape));
    return Collections.singletonList(new Pair<>(array5d,
                    "get5dReshapedWithShape(" + seed + "," + Arrays.toString(shape) + ").get(0)"));
}
 

public ConditionBuilder and(Condition... conditions) {
    if (soFar == null)
        soFar = new And(conditions);
    else {
        soFar = new And(ArrayUtil.combine(conditions, new Condition[] {soFar}));
    }
    return this;
}
 

protected void addArgs() {
    addIArgument(config.getK(),
            config.getS(),
            config.getP(),
            ArrayUtil.fromBoolean(config.isSameMode()),
            ArrayUtil.fromBoolean(config.isNHWC()));
}