Java Code Examples for net.jpountz.lz4.LZ4Compressor#maxCompressedLength()

The following examples show how to use net.jpountz.lz4.LZ4Compressor#maxCompressedLength() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LZ4Codec.java    From redisson with Apache License 2.0 6 votes vote down vote up 
@Override
public ByteBuf encode(Object in) throws IOException {
    ByteBuf bytes = null;
    try {
        LZ4Compressor compressor = factory.fastCompressor();
        bytes = innerCodec.getValueEncoder().encode(in);
        ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes());
        
        int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes());
        ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE);
        out.writeInt(bytes.readableBytes());
        ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes());
        int pos = outBuf.position();
        
        compressor.compress(srcBuf, outBuf);
        
        int compressedLength = outBuf.position() - pos;
        out.writerIndex(out.writerIndex() + compressedLength);
        return out;
    } finally {
        if (bytes != null) {
            bytes.release();
        }
    }
}
Example 2
Source File: LZ4CompressorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
Example 3
Source File: ChannelLZ4Compressor.java    From datakernel with Apache License 2.0 5 votes vote down vote up 
private static ByteBuf compressBlock(LZ4Compressor compressor, StreamingXXHash32 checksum, byte[] bytes, int off, int len) {
	checkArgument(len != 0);

	int compressionLevel = compressionLevel(max(len, MIN_BLOCK_SIZE));

	int outputBufMaxSize = HEADER_LENGTH + ((compressor == null) ? len : compressor.maxCompressedLength(len));
	ByteBuf outputBuf = ByteBufPool.allocate(outputBufMaxSize);
	outputBuf.put(MAGIC);

	byte[] outputBytes = outputBuf.array();

	checksum.reset();
	checksum.update(bytes, off, len);
	int check = checksum.getValue();

	int compressedLength = len;
	if (compressor != null) {
		compressedLength = compressor.compress(bytes, off, len, outputBytes, HEADER_LENGTH);
	}

	int compressMethod;
	if (compressor == null || compressedLength >= len) {
		compressMethod = COMPRESSION_METHOD_RAW;
		compressedLength = len;
		System.arraycopy(bytes, off, outputBytes, HEADER_LENGTH, len);
	} else {
		compressMethod = COMPRESSION_METHOD_LZ4;
	}

	outputBytes[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
	writeIntLE(compressedLength, outputBytes, MAGIC_LENGTH + 1);
	writeIntLE(len, outputBytes, MAGIC_LENGTH + 5);
	writeIntLE(check, outputBytes, MAGIC_LENGTH + 9);

	outputBuf.tail(HEADER_LENGTH + compressedLength);

	return outputBuf;
}
Example 4
Source File: LZ4CompressorTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
Example 5
Source File: ODAGPartLZ4Wrapper.java    From Arabesque with Apache License 2.0 5 votes vote down vote up 
private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}
Example 6
Source File: ODAGStashLZ4Wrapper.java    From Arabesque with Apache License 2.0 5 votes vote down vote up 
private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}
Example 7
Source File: LZ4ObjectCache.java    From Arabesque with Apache License 2.0 5 votes vote down vote up 
private void compressDataOutput() {
    if (!Configuration.get().isUseCompressedCaches() || !uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}