Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#data()

@Test
public void testQRSquare() {
    INDArray A = Nd4j.create(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);
    INDArray R = Nd4j.create(A.dataType(), A.columns(), A.columns());

    Nd4j.getBlasWrapper().lapack().geqrf(A, R);

    A.mmuli(R);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

@Test
public void testCholeskyL() {
    INDArray A = Nd4j.create(new double[] {2, -1, 1, -1, 2, -1, 1, -1, 2,});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, true);

    A.mmuli(A.transpose());
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

public NumpyArray(INDArray nd4jArray) {
    Nd4j.getAffinityManager().ensureLocation(nd4jArray, AffinityManager.Location.HOST);
    DataBuffer buff = nd4jArray.data();
    address = buff.pointer().address();
    shape = nd4jArray.shape();
    long[] nd4jStrides = nd4jArray.stride();
    strides = new long[nd4jStrides.length];
    int elemSize = buff.getElementSize();
    for (int i = 0; i < strides.length; i++) {
        strides[i] = nd4jStrides[i] * elemSize;
    }
    dtype = nd4jArray.dataType();
    this.nd4jArray = nd4jArray;
    String cacheKey = address + "_" + nd4jArray.length() + "_" + dtype + "_" + ArrayUtils.toString(strides);
    arrayCache.put(cacheKey, nd4jArray);
}
 

@Test
public void testCholeskyL() {
    INDArray A = Nd4j.create(new double[] {2, -1, 1, -1, 2, -1, 1, -1, 2,});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, true);

    A.mmuli(A.transpose());
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

@Test
public void testQRRect() {
    INDArray A = Nd4j.create(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
    A = A.reshape('f', 4, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    INDArray R = Nd4j.create(A.columns(), A.columns());
    Nd4j.getBlasWrapper().lapack().geqrf(A, R);

    A.mmuli(R);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        allocated.rewind();
}
 

@Override
public CudaContext prepareActionAllWrite(INDArray... operands) {
    CudaContext context = (CudaContext) allocator.getDeviceContext().getContext();
    int cId = allocator.getDeviceId();

    for (INDArray operand : operands) {
        if (operand == null)
            continue;

        Nd4j.getCompressor().autoDecompress(operand);

        AllocationPoint pointData = allocator.getAllocationPoint(operand);
        AllocationPoint pointShape = allocator.getAllocationPoint(operand.shapeInfoDataBuffer());

        pointData.acquireLock();

        if (pointData.getDeviceId() != cId && pointData.getDeviceId() >= 0) {
            DataBuffer buffer = operand.data().originalDataBuffer() == null ? operand.data()
                            : operand.data().originalDataBuffer();
            allocator.getMemoryHandler().relocateObject(buffer);
        }

        if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) {
            ((JCublasNDArray) operand).setShapeInfoDataBuffer(
                            Nd4j.getConstantHandler().relocateConstantSpace(operand.shapeInfoDataBuffer()));
        }

        prepareDelayedMemory(operand);
        allocator.getAllocationPoint(operand).setCurrentContext(context);
    }
    return context;
}
 

@Test
public void testCholeskyU() {
    INDArray A = Nd4j.create(new double[] {3, -1, 2, -1, 3, -1, 2, -1, 3,});
    A = A.reshape('f', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, false);
    A = A.transpose().mmul(A);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

@Override
public INDArray nextFloat(char order, long[] shape) {
    long length = ArrayUtil.prodLong(shape);
    INDArray ret = Nd4j.create(shape, order);

    DataBuffer data = ret.data();
    for (long i = 0; i < length; i++) {
        data.put(i, nextFloat());
    }

    return ret;
}
 

/**
 * in place decompression of the given
 * ndarray. If the ndarray isn't compressed
 * this will do nothing
 * @param array the array to decompressed
 *              if it is comprssed
 */
public void decompressi(INDArray array) {
    if (array.data().dataType() != DataType.COMPRESSED)
        return;

    val comp = (CompressedDataBuffer) array.data();
    val descriptor = comp.getCompressionDescriptor();


    if (!codecs.containsKey(descriptor.getCompressionAlgorithm()))
        throw new RuntimeException("Non-existent compression algorithm requested: ["
                        + descriptor.getCompressionAlgorithm() + "]");

    codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array);
}
 

@Override
public void tickHostWrite(INDArray array) {
    DataBuffer buffer =
                    array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();

    tickHostWrite(buffer);
}
 

@Test
public void testCholeskyU() {
    INDArray A = Nd4j.create(new double[] {2, -1, 2, -1, 2, -1, 2, -1, 2,});
    A = A.reshape('f', 3, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, false);
    A = A.transpose().mmul(A);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 

@Override
public CudaContext prepareActionAllWrite(INDArray... operands) {
    val context = allocator.getDeviceContext();
    val cId = allocator.getDeviceId();

    for (INDArray operand : operands) {
        if (operand == null || operand.isEmpty())
            continue;

        Nd4j.getCompressor().autoDecompress(operand);

        val pointData = allocator.getAllocationPoint(operand);
        val pointShape = allocator.getAllocationPoint(operand.shapeInfoDataBuffer());


        if (pointData.getDeviceId() != cId && pointData.getDeviceId() >= 0) {
            DataBuffer buffer = operand.data().originalDataBuffer() == null ? operand.data()
                            : operand.data().originalDataBuffer();
            allocator.getMemoryHandler().relocateObject(buffer);
        }

        if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) {
            ((JCublasNDArray) operand).setShapeInfoDataBuffer(
                            Nd4j.getConstantHandler().relocateConstantSpace(operand.shapeInfoDataBuffer()));
        }

        prepareDelayedMemory(operand);
        allocator.getAllocationPoint(operand).setCurrentContext(context);
    }
    return context;
}
 

@Override
public INDArray nextDouble(char order, long[] shape) {
    long length = ArrayUtil.prodLong(shape);
    INDArray ret = Nd4j.create(shape, order);

    DataBuffer data = ret.data();
    for (long i = 0; i < length; i++) {
        data.put(i, nextDouble());
    }

    return ret;
}
 

/**
 * This method should be called to make sure that data on host side is actualized
 *
 * @param array
 */
@Override
public void synchronizeHostData(INDArray array) {
    if (array.isEmpty() || array.isS())
        return;

    val buffer = array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();
    synchronizeHostData(buffer);
}
 

/**
 *
 * @param array
 * @return
 */
public INDArray decompress(INDArray array) {
    if (array.data().dataType() != DataType.COMPRESSED)
        return array;

    CompressedDataBuffer comp = (CompressedDataBuffer) array.data();
    CompressionDescriptor descriptor = comp.getCompressionDescriptor();

    if (!codecs.containsKey(descriptor.getCompressionAlgorithm()))
        throw new RuntimeException("Non-existent compression algorithm requested: ["
                        + descriptor.getCompressionAlgorithm() + "]");

    return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array);
}
 

@Override
public AllocationPoint getAllocationPoint(INDArray array) {
    DataBuffer buffer =
                    array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();
    return getAllocationPoint(buffer);
}
 

@Test
public void testAsBytes() {
    INDArray orig = Nd4j.linspace(DataType.INT, 0, 10, 1);

    for (DataType dt : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.BFLOAT16,
            DataType.LONG, DataType.INT, DataType.SHORT, DataType.BYTE, DataType.BOOL,
            DataType.UINT64, DataType.UINT32, DataType.UINT16, DataType.UBYTE}) {
        INDArray arr = orig.castTo(dt);

        byte[] b = arr.data().asBytes();        //NOTE: BIG ENDIAN

        if(ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
            //Switch from big endian (as defined by asBytes which uses big endian) to little endian
            int w = dt.width();
            if (w > 1) {
                int len = b.length / w;
                for (int i = 0; i < len; i++) {
                    for (int j = 0; j < w / 2; j++) {
                        byte temp = b[(i + 1) * w - j - 1];
                        b[(i + 1) * w - j - 1] = b[i * w + j];
                        b[i * w + j] = temp;
                    }
                }
            }
        }

        INDArray arr2 = Nd4j.create(dt, arr.shape());
        ByteBuffer bb = arr2.data().pointer().asByteBuffer();
        bb.position(0);
        bb.put(b);

        Nd4j.getAffinityManager().tagLocation(arr2, AffinityManager.Location.HOST);

        assertEquals(arr.toString(), arr2.toString());
        assertEquals(arr, arr2);

        //Sanity check on data buffer getters:
        DataBuffer db = arr.data();
        DataBuffer db2 = arr2.data();
        for( int i=0; i<10; i++ ){
            assertEquals(db.getDouble(i), db2.getDouble(i), 0);
            assertEquals(db.getFloat(i), db2.getFloat(i), 0);
            assertEquals(db.getInt(i), db2.getInt(i), 0);
            assertEquals(db.getLong(i), db2.getLong(i), 0);
            assertEquals(db.getNumber(i), db2.getNumber(i));
        }

        assertArrayEquals(db.getDoublesAt(0, 10), db2.getDoublesAt(0, 10), 0);
        assertArrayEquals(db.getFloatsAt(0, 10), db2.getFloatsAt(0, 10), 0);
        assertArrayEquals(db.getIntsAt(0, 10), db2.getIntsAt(0, 10));
        assertArrayEquals(db.getLongsAt(0, 10), db2.getLongsAt(0, 10));
    }
}
 

public BytePointer toBytePointer() throws PythonException{
        if (Python.isinstance(this, Python.bytesType())){
            PyObject byteArray = PyByteArray_FromObject(nativePythonObject);
            return PyByteArray_AsString(byteArray);

        }
        else if (Python.isinstance(this, Python.bytearrayType())){
            return PyByteArray_AsString(nativePythonObject);
        }
        else if (Python.isinstance(this, Python.memoryviewType())){

//            PyObject np = PyImport_ImportModule("numpy");
//            PyObject array = PyObject_GetAttrString(np, "asarray");
//            PyObject npArr = PyObject_CallObject(array, nativePythonObject); // Doesn't work
            // Invoke interpreter:
            String tempContext = "temp" + UUID.randomUUID().toString().replace('-', '_');
            String originalContext = Python.getCurrentContext();
            Python.setContext(tempContext);
            PythonExecutioner.setVariable("memview", this);
            PythonExecutioner.exec("import numpy as np\narr = np.frombuffer(memview, dtype='int8')");
            INDArray arr = PythonExecutioner.getVariable("arr").toNumpy().getNd4jArray();
            if(arr.data() instanceof BaseDataBuffer){
                ((BaseDataBuffer)arr.data()).syncToPrimary();
            }
            BytePointer ret = new BytePointer(arr.data().pointer());
            Python.setContext(originalContext);
            Python.deleteContext(tempContext);
            return ret;
        } else {
            PyObject ctypes = PyImport_ImportModule("ctypes");
            PyObject cArrType = PyObject_GetAttrString(ctypes, "Array");
            if (PyObject_IsInstance(nativePythonObject, cArrType) != 0){
                PyObject cVoidP = PyObject_GetAttrString(ctypes, "c_void_p");
                PyObject cast = PyObject_GetAttrString(ctypes, "cast");
                PyObject argsTuple = PyTuple_New(2);
                PyTuple_SetItem(argsTuple, 0, nativePythonObject);
                PyTuple_SetItem(argsTuple, 1, cVoidP);
                PyObject voidPtr = PyObject_Call(cast, argsTuple, null);
                PyObject pyAddress = PyObject_GetAttrString(voidPtr, "value");
                long address = PyLong_AsLong(pyAddress);
                long size = PyObject_Size(nativePythonObject);
                Py_DecRef(ctypes);
                Py_DecRef(cArrType);
                Py_DecRef(argsTuple);
                Py_DecRef(voidPtr);
                Py_DecRef(pyAddress);
                Pointer ptr = NativeOpsHolder.getInstance().getDeviceNativeOps().pointerForAddress(address);
                ptr = ptr.limit(size);
                ptr = ptr.capacity(size);
                return new BytePointer(ptr);
            }
            else{
                throw new PythonException("Expected bytes, bytearray, memoryview or ctypesArray. Received " + Python.type(this).toString());
            }
        }
    }
 

/**
 * Create a {@link Buffer}
 * representing the location metadata of the actual data
 * contents for the ndarrays' {@link DataBuffer}
 * @param bufferBuilder the buffer builder in use
 * @param arr the array to add the underlying data for
 * @return the offset added
 */
public static int addDataForArr(FlatBufferBuilder bufferBuilder, INDArray arr) {
    DataBuffer toAdd = arr.isView() ? arr.dup().data() : arr.data();
    int offset = DataBufferStruct.createDataBufferStruct(bufferBuilder,toAdd);
    int ret = Buffer.createBuffer(bufferBuilder,offset,toAdd.length() * toAdd.getElementSize());
    return ret;

}