org.nd4j.linalg.compression.CompressedDataBuffer Java Exaples

Source File: Int8.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length);

    return buffer;
}

Source File: BinarySerde.java From deeplearning4j with Apache License 2.0

6 votes

/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        ((Buffer) allocated).rewind();
}

Source File: Gzip.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    try {
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(stream);
        DataOutputStream dos = new DataOutputStream(gzip);

        buffer.write(dos);
        dos.flush();
        dos.close();

        byte[] bytes = stream.toByteArray();
        //            logger.info("Bytes: {}", Arrays.toString(bytes));
        BytePointer pointer = new BytePointer(bytes);
        CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
        descriptor.setCompressedLength(bytes.length);

        CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor);

        return result;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Source File: Gzip.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public DataBuffer decompress(DataBuffer buffer, DataType dataType) {
    try {

        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();

        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);

        DataBuffer bufferRestored = Nd4j.createBuffer(dataType, descriptor.getNumberOfElements(), false);
        BaseDataBuffer.readHeader(dis);
        bufferRestored.read(dis, DataBuffer.AllocationMode.MIXED_DATA_TYPES, descriptor.getNumberOfElements(), dataType);

        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Source File: NoOp.java From deeplearning4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataTypeEx srcType, Pointer srcPointer, int length,
                                               int elementSize) {

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressionType(getCompressionType());
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setCompressedLength(length * elementSize);
    descriptor.setNumberOfElements(length);

    BytePointer ptr = new BytePointer(length * elementSize);

    val perfD = PerformanceTracker.getInstance().helperStartTransaction();

    // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
    Pointer.memcpy(ptr, srcPointer, length * elementSize);

    PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST);

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    return buffer;
}

Source File: CpuNDArrayFactory.java From deeplearning4j with Apache License 2.0

6 votes

/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataTypeEx typeSrc, INDArray source, DataTypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}

Source File: JCublasNDArrayFactory.java From deeplearning4j with Apache License 2.0

6 votes

/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataTypeEx typeSrc, INDArray source, DataTypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}

Source File: BinarySerde.java From nd4j with Apache License 2.0

6 votes

/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        allocated.rewind();
}

Source File: Int16.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length * 2);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 2);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length);

    return buffer;
}

Source File: Float8.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length);

    return buffer;
}

Source File: Gzip.java From nd4j with Apache License 2.0

6 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    try {
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(stream);
        DataOutputStream dos = new DataOutputStream(gzip);

        buffer.write(dos);
        dos.flush();
        dos.close();

        byte[] bytes = stream.toByteArray();
        //            logger.info("Bytes: {}", Arrays.toString(bytes));
        BytePointer pointer = new BytePointer(bytes);
        CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
        descriptor.setCompressedLength(bytes.length);

        CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor);

        return result;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Source File: Gzip.java From nd4j with Apache License 2.0

6 votes

@Override
public DataBuffer decompress(DataBuffer buffer) {
    try {

        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();

        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);

        DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements());
        bufferRestored.read(dis);

        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Source File: Float16.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length * 2);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 2);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length);

    return buffer;
}

Source File: NoOp.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressionType(getCompressionType());
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setCompressedLength(length * elementSize);
    descriptor.setNumberOfElements(length);

    BytePointer ptr = new BytePointer(length * elementSize);

    val perfD = PerformanceTracker.getInstance().helperStartTransaction();

    // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
    Pointer.memcpy(ptr, srcPointer, length * elementSize);

    PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST);

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    return buffer;
}

Source File: Uint8.java From nd4j with Apache License 2.0

6 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length);

    return buffer;
}

Source File: JCublasNDArrayFactory.java From nd4j with Apache License 2.0

6 votes

/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataBuffer.TypeEx typeSrc, INDArray source, DataBuffer.TypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}

Source File: CpuNDArrayFactory.java From nd4j with Apache License 2.0

6 votes

/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataBuffer.TypeEx typeSrc, INDArray source, DataBuffer.TypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}

Source File: CudaFlexibleThreshold.java From nd4j with Apache License 2.0

5 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}));
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 3;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}

Source File: AbstractCompressor.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public INDArray decompress(INDArray array) {
    if (!array.isCompressed())
        return array;

    val descriptor = ((CompressedDataBuffer)array.data()).getCompressionDescriptor();
    val buffer = decompress(array.data(), descriptor.getOriginalDataType());
    val shapeInfo = array.shapeInfoDataBuffer();
    INDArray rest = Nd4j.createArrayFromShapeBuffer(buffer, shapeInfo);

    return rest;
}

Source File: AbstractCompressor.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public void decompressi(INDArray array) {
    if (!array.isCompressed())
        return;

    array.markAsCompressed(false);
    array.setData(decompress(array.data(), ((CompressedDataBuffer)array.data()).getCompressionDescriptor().getOriginalDataType()));
}

Source File: AtomicAllocator.java From nd4j with Apache License 2.0

5 votes

@Override
public AllocationPoint getAllocationPoint(DataBuffer buffer) {
    if (buffer instanceof CompressedDataBuffer) {
        log.warn("Trying to get AllocationPoint from CompressedDataBuffer");
        throw new RuntimeException("AP CDB");
    }

    return getAllocationPoint(buffer.getTrackingPoint());
}

Source File: CpuThreshold.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new long[]{1, buffer.length()}, buffer.dataType()).getFirst());
    MatchCondition condition = new MatchCondition(temp, Conditions.absGreaterThanOrEqual(threshold));
    int cntAbs = Nd4j.getExecutioner().exec(condition).getInt(0);


    //log.info("density ratio: {}", String.format("%.2f", cntAbs * 100.0f / buffer.length()));

    if (cntAbs < 2)
        return null;

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold));
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());



    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataTypeEx.THRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}

Source File: CpuFlexibleThreshold.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new long[]{1, buffer.length()}, DataType.INT).getFirst());
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataTypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}

Source File: CpuFlexibleThreshold.java From nd4j with Apache License 2.0

5 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}).getFirst());
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}

Source File: CpuThreshold.java From nd4j with Apache License 2.0

5 votes

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}).getFirst());
    MatchCondition condition = new MatchCondition(temp, Conditions.absGreaterThanOrEqual(threshold));
    int cntAbs = Nd4j.getExecutioner().exec(condition, Integer.MAX_VALUE).getInt(0);


    //log.info("density ratio: {}", String.format("%.2f", cntAbs * 100.0f / buffer.length()));

    if (cntAbs < 2)
        return null;

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold));
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());



    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.THRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}

Source File: JCublasNDArrayFactory.java From deeplearning4j with Apache License 2.0

4 votes

@Override
public void convertDataEx(DataTypeEx typeSrc, Pointer source, DataTypeEx typeDst, DataBuffer buffer) {
    Pointer srcPtr = null;
    Pointer dstPtr = null;
    long size = 0;
    long ssize = 0;
    val stream = AtomicAllocator.getInstance().getDeviceContext().getOldStream();
    if (buffer instanceof CompressedDataBuffer) {
        // compressing
        size = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getCompressedLength();
        ssize = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getOriginalLength();

        srcPtr = nativeOps.mallocDevice(ssize, 0, 0);
        dstPtr = nativeOps.mallocDevice(size, 0, 0);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());

        nativeOps.memcpyAsync(srcPtr, source, ssize, CudaConstants.cudaMemcpyHostToDevice, stream);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());
    } else {
        // decompressing
        throw new UnsupportedOperationException();
    }

    convertDataEx(typeSrc, srcPtr, typeDst, dstPtr, buffer.length());
    nativeOps.memcpyAsync(buffer.addressPointer(), dstPtr, size, CudaConstants.cudaMemcpyHostToHost, stream);

    stream.synchronize();

    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    if (buffer instanceof CompressedDataBuffer) {
        nativeOps.freeDevice(srcPtr, 0);
        nativeOps.freeDevice(dstPtr, 0);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());
    }
}

Source File: CpuThreshold.java From deeplearning4j with Apache License 2.0

4 votes

@Override
protected CompressedDataBuffer compressPointer(DataTypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}

Source File: CpuThreshold.java From nd4j with Apache License 2.0

4 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}

Source File: AbstractCompressor.java From nd4j with Apache License 2.0

4 votes

protected abstract CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
int elementSize);

Source File: CudaThreshold.java From nd4j with Apache License 2.0

4 votes

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}

org.nd4j.linalg.compression.CompressedDataBuffer Java Examples