org.nd4j.linalg.compression.CompressedDataBuffer Java Examples

The following examples show how to use org.nd4j.linalg.compression.CompressedDataBuffer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Int8.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length);

    return buffer;
}
 
Example #2
Source File: BinarySerde.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        ((Buffer) allocated).rewind();
}
 
Example #3
Source File: Gzip.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    try {
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(stream);
        DataOutputStream dos = new DataOutputStream(gzip);

        buffer.write(dos);
        dos.flush();
        dos.close();

        byte[] bytes = stream.toByteArray();
        //            logger.info("Bytes: {}", Arrays.toString(bytes));
        BytePointer pointer = new BytePointer(bytes);
        CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
        descriptor.setCompressedLength(bytes.length);

        CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor);

        return result;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #4
Source File: Gzip.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
public DataBuffer decompress(DataBuffer buffer, DataType dataType) {
    try {

        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();

        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);

        DataBuffer bufferRestored = Nd4j.createBuffer(dataType, descriptor.getNumberOfElements(), false);
        BaseDataBuffer.readHeader(dis);
        bufferRestored.read(dis, DataBuffer.AllocationMode.MIXED_DATA_TYPES, descriptor.getNumberOfElements(), dataType);

        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #5
Source File: NoOp.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataTypeEx srcType, Pointer srcPointer, int length,
                                               int elementSize) {

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressionType(getCompressionType());
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setCompressedLength(length * elementSize);
    descriptor.setNumberOfElements(length);

    BytePointer ptr = new BytePointer(length * elementSize);

    val perfD = PerformanceTracker.getInstance().helperStartTransaction();

    // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
    Pointer.memcpy(ptr, srcPointer, length * elementSize);

    PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST);

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    return buffer;
}
 
Example #6
Source File: CpuNDArrayFactory.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataTypeEx typeSrc, INDArray source, DataTypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}
 
Example #7
Source File: JCublasNDArrayFactory.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataTypeEx typeSrc, INDArray source, DataTypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}
 
Example #8
Source File: BinarySerde.java    From nd4j with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        allocated.rewind();
}
 
Example #9
Source File: Int16.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length * 2);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 2);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length);

    return buffer;
}
 
Example #10
Source File: Float8.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length);

    return buffer;
}
 
Example #11
Source File: Gzip.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    try {
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(stream);
        DataOutputStream dos = new DataOutputStream(gzip);

        buffer.write(dos);
        dos.flush();
        dos.close();

        byte[] bytes = stream.toByteArray();
        //            logger.info("Bytes: {}", Arrays.toString(bytes));
        BytePointer pointer = new BytePointer(bytes);
        CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
        descriptor.setCompressedLength(bytes.length);

        CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor);

        return result;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #12
Source File: Gzip.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
public DataBuffer decompress(DataBuffer buffer) {
    try {

        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();

        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);

        DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements());
        bufferRestored.read(dis);

        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #13
Source File: Float16.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length * 2);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 2);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length);

    return buffer;
}
 
Example #14
Source File: NoOp.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressionType(getCompressionType());
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setCompressedLength(length * elementSize);
    descriptor.setNumberOfElements(length);

    BytePointer ptr = new BytePointer(length * elementSize);

    val perfD = PerformanceTracker.getInstance().helperStartTransaction();

    // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
    Pointer.memcpy(ptr, srcPointer, length * elementSize);

    PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST);

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    return buffer;
}
 
Example #15
Source File: Uint8.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
                int elementSize) {

    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length);

    return buffer;
}
 
Example #16
Source File: JCublasNDArrayFactory.java    From nd4j with Apache License 2.0 6 votes vote down vote up
/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataBuffer.TypeEx typeSrc, INDArray source, DataBuffer.TypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}
 
Example #17
Source File: CpuNDArrayFactory.java    From nd4j with Apache License 2.0 6 votes vote down vote up
/**
 * This method converts Single/Double precision databuffer to Half-precision databuffer
 *
 * @param typeSrc
 * @param source
 * @param typeDst @return
 */
@Override
public INDArray convertDataEx(DataBuffer.TypeEx typeSrc, INDArray source, DataBuffer.TypeEx typeDst) {
    if (source.isView())
        throw new UnsupportedOperationException("Impossible to compress View. Consider using dup() before. ");

    DataBuffer buffer = convertDataEx(typeSrc, source.data(), typeDst);
    source.setData(buffer);

    if (buffer instanceof CompressedDataBuffer)
        source.markAsCompressed(true);
    else
        source.markAsCompressed(false);

    return source;
}
 
Example #18
Source File: CudaFlexibleThreshold.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}));
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 3;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}
 
Example #19
Source File: AbstractCompressor.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public INDArray decompress(INDArray array) {
    if (!array.isCompressed())
        return array;

    val descriptor = ((CompressedDataBuffer)array.data()).getCompressionDescriptor();
    val buffer = decompress(array.data(), descriptor.getOriginalDataType());
    val shapeInfo = array.shapeInfoDataBuffer();
    INDArray rest = Nd4j.createArrayFromShapeBuffer(buffer, shapeInfo);

    return rest;
}
 
Example #20
Source File: AbstractCompressor.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public void decompressi(INDArray array) {
    if (!array.isCompressed())
        return;

    array.markAsCompressed(false);
    array.setData(decompress(array.data(), ((CompressedDataBuffer)array.data()).getCompressionDescriptor().getOriginalDataType()));
}
 
Example #21
Source File: AtomicAllocator.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public AllocationPoint getAllocationPoint(DataBuffer buffer) {
    if (buffer instanceof CompressedDataBuffer) {
        log.warn("Trying to get AllocationPoint from CompressedDataBuffer");
        throw new RuntimeException("AP CDB");
    }

    return getAllocationPoint(buffer.getTrackingPoint());
}
 
Example #22
Source File: CpuThreshold.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new long[]{1, buffer.length()}, buffer.dataType()).getFirst());
    MatchCondition condition = new MatchCondition(temp, Conditions.absGreaterThanOrEqual(threshold));
    int cntAbs = Nd4j.getExecutioner().exec(condition).getInt(0);


    //log.info("density ratio: {}", String.format("%.2f", cntAbs * 100.0f / buffer.length()));

    if (cntAbs < 2)
        return null;

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold));
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());



    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataTypeEx.THRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}
 
Example #23
Source File: CpuFlexibleThreshold.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new long[]{1, buffer.length()}, DataType.INT).getFirst());
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataTypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}
 
Example #24
Source File: CpuFlexibleThreshold.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}).getFirst());
    double max = temp.amaxNumber().doubleValue();

    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold)); // please note, this value will be ovewritten anyway
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());

    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}
 
Example #25
Source File: CpuThreshold.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[]{1, (int) buffer.length()}).getFirst());
    MatchCondition condition = new MatchCondition(temp, Conditions.absGreaterThanOrEqual(threshold));
    int cntAbs = Nd4j.getExecutioner().exec(condition, Integer.MAX_VALUE).getInt(0);


    //log.info("density ratio: {}", String.format("%.2f", cntAbs * 100.0f / buffer.length()));

    if (cntAbs < 2)
        return null;

    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 4;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    pointer.put(2, Float.floatToIntBits(threshold));
    pointer.put(3, 0);

    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(compressedLength * 4); // sizeOf(INT)
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());

    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());



    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);

    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.THRESHOLD, pointer, buffer.length());

    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);

    return cbuff;
}
 
Example #26
Source File: JCublasNDArrayFactory.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Override
public void convertDataEx(DataTypeEx typeSrc, Pointer source, DataTypeEx typeDst, DataBuffer buffer) {
    Pointer srcPtr = null;
    Pointer dstPtr = null;
    long size = 0;
    long ssize = 0;
    val stream = AtomicAllocator.getInstance().getDeviceContext().getOldStream();
    if (buffer instanceof CompressedDataBuffer) {
        // compressing
        size = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getCompressedLength();
        ssize = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getOriginalLength();

        srcPtr = nativeOps.mallocDevice(ssize, 0, 0);
        dstPtr = nativeOps.mallocDevice(size, 0, 0);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());

        nativeOps.memcpyAsync(srcPtr, source, ssize, CudaConstants.cudaMemcpyHostToDevice, stream);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());
    } else {
        // decompressing
        throw new UnsupportedOperationException();
    }

    convertDataEx(typeSrc, srcPtr, typeDst, dstPtr, buffer.length());
    nativeOps.memcpyAsync(buffer.addressPointer(), dstPtr, size, CudaConstants.cudaMemcpyHostToHost, stream);

    stream.synchronize();

    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    if (buffer instanceof CompressedDataBuffer) {
        nativeOps.freeDevice(srcPtr, 0);
        nativeOps.freeDevice(dstPtr, 0);

        if (nativeOps.lastErrorCode() != 0)
            throw new RuntimeException(nativeOps.lastErrorMessage());
    }
}
 
Example #27
Source File: CpuThreshold.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataTypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}
 
Example #28
Source File: CpuThreshold.java    From nd4j with Apache License 2.0 4 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}
 
Example #29
Source File: AbstractCompressor.java    From nd4j with Apache License 2.0 4 votes vote down vote up
protected abstract CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length,
int elementSize);
 
Example #30
Source File: CudaThreshold.java    From nd4j with Apache License 2.0 4 votes vote down vote up
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    throw new UnsupportedOperationException();
}