net.jpountz.lz4.LZ4Compressor#compress

Source File: SerializationUtils.java From exchange-core with Apache License 2.0

6 votes

public static long[] bytesToLongArrayLz4(final LZ4Compressor lz4Compressor, final NativeBytes<Void> bytes, final int padding) {
        int originalSize = (int) bytes.readRemaining();
//        log.debug("COMPRESS originalSize={}", originalSize);

        final ByteBuffer byteBuffer = ByteBuffer.allocate(originalSize);

        bytes.read(byteBuffer);

        byteBuffer.flip();

        final ByteBuffer byteBufferCompressed = ByteBuffer.allocate(4 + lz4Compressor.maxCompressedLength(originalSize));
        byteBufferCompressed.putInt(originalSize);// override with compressed length
        lz4Compressor.compress(byteBuffer, byteBufferCompressed);

        byteBufferCompressed.flip();

        int compressedBytesLen = byteBufferCompressed.remaining();

        return toLongsArray(
                byteBufferCompressed.array(),
                byteBufferCompressed.arrayOffset(),
                compressedBytesLen,
                padding);
    }

Source File: Compressor.java From vespa with Apache License 2.0

6 votes

public long warmup(double seconds) {
    byte [] input = new byte[0x4000];
    new Random().nextBytes(input);
    long timeDone = System.nanoTime() + (long)(seconds*1000000000);
    long compressedBytes = 0;
    byte [] decompressed = new byte [input.length];
    LZ4FastDecompressor fastDecompressor = factory.fastDecompressor();
    LZ4SafeDecompressor safeDecompressor = factory.safeDecompressor();
    LZ4Compressor fastCompressor = factory.fastCompressor();
    LZ4Compressor highCompressor = factory.highCompressor();
    while (System.nanoTime() < timeDone) {
        byte [] compressedFast = fastCompressor.compress(input);
        byte [] compressedHigh = highCompressor.compress(input);
        fastDecompressor.decompress(compressedFast, decompressed);
        fastDecompressor.decompress(compressedHigh, decompressed);
        safeDecompressor.decompress(compressedFast, decompressed);
        safeDecompressor.decompress(compressedHigh, decompressed);
        compressedBytes += compressedFast.length + compressedHigh.length;
    }
    return compressedBytes;
}

Source File: LZ4Codec.java From redisson with Apache License 2.0

6 votes

@Override
public ByteBuf encode(Object in) throws IOException {
    ByteBuf bytes = null;
    try {
        LZ4Compressor compressor = factory.fastCompressor();
        bytes = innerCodec.getValueEncoder().encode(in);
        ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes());
        
        int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes());
        ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE);
        out.writeInt(bytes.readableBytes());
        ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes());
        int pos = outBuf.position();
        
        compressor.compress(srcBuf, outBuf);
        
        int compressedLength = outBuf.position() - pos;
        out.writerIndex(out.writerIndex() + compressedLength);
        return out;
    } finally {
        if (bytes != null) {
            bytes.release();
        }
    }
}

Source File: CompressionUtil.java From consulo with Apache License 2.0

6 votes

public static int writeCompressed(@Nonnull DataOutput out, @Nonnull byte[] bytes, int start, int length) throws IOException {
  if (length > COMPRESSION_THRESHOLD) {
    LZ4Compressor compressor = compressor();

    byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
    int compressedSize = compressor.compress(bytes, start, length, compressedOutputBuffer, 0);
    if (compressedSize < length) {
      DataInputOutputUtil.writeINT(out, -compressedSize);
      DataInputOutputUtil.writeINT(out, length - compressedSize);
      out.write(compressedOutputBuffer, 0, compressedSize);
      return compressedSize;
    }
  }
  DataInputOutputUtil.writeINT(out, length);
  out.write(bytes, start, length);
  return length;
}

Source File: CompressionUtil.java From consulo with Apache License 2.0

6 votes

public static int writeCompressedWithoutOriginalBufferLength(@Nonnull DataOutput out, @Nonnull byte[] bytes, int length) throws IOException {
  long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;

  LZ4Compressor compressor = compressor();
  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
  int compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && (requests & 0x1fff) == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}

Source File: LZ4CompressorTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }

Source File: CompressedBuffedReaderTest.java From ClickHouse-Native-JDBC with Apache License 2.0

5 votes

private byte[] compressedData(byte[] bytes) {
    LZ4Compressor lz4Compressor = LZ4Factory.safeInstance().fastCompressor();
    byte[] compressData = lz4Compressor.compress(bytes);
    byte[] data = new byte[compressData.length + 9 + 16];

    data[16] = (byte) (0x82 & 0xFF);
    System.arraycopy(compressData, 0, data, 9 + 16, compressData.length);
    System.arraycopy(littleEndian(compressData.length + 9), 0, data, 17, 4);
    System.arraycopy(littleEndian(bytes.length), 0, data, 21, 4);

    return data;
}

Source File: CompressionProcessorImpl.java From ignite with Apache License 2.0

5 votes

/**
 * @param compactPage Compacted page.
 * @param compactSize Compacted page size.
 * @param compressLevel Compression level.
 * @return Compressed page.
 */
private ByteBuffer compressPageLz4(ByteBuffer compactPage, int compactSize, int compressLevel) {
    LZ4Compressor compressor = Lz4.getCompressor(compressLevel);

    ByteBuffer compressedPage = compressBuf.get();

    copyPageHeader(compactPage, compressedPage, compactSize);
    compressor.compress(compactPage, compressedPage);

    compactPage.flip();
    compressedPage.flip();

    return compressedPage;
}

Source File: ChannelLZ4Compressor.java From datakernel with Apache License 2.0

5 votes

private static ByteBuf compressBlock(LZ4Compressor compressor, StreamingXXHash32 checksum, byte[] bytes, int off, int len) {
	checkArgument(len != 0);

	int compressionLevel = compressionLevel(max(len, MIN_BLOCK_SIZE));

	int outputBufMaxSize = HEADER_LENGTH + ((compressor == null) ? len : compressor.maxCompressedLength(len));
	ByteBuf outputBuf = ByteBufPool.allocate(outputBufMaxSize);
	outputBuf.put(MAGIC);

	byte[] outputBytes = outputBuf.array();

	checksum.reset();
	checksum.update(bytes, off, len);
	int check = checksum.getValue();

	int compressedLength = len;
	if (compressor != null) {
		compressedLength = compressor.compress(bytes, off, len, outputBytes, HEADER_LENGTH);
	}

	int compressMethod;
	if (compressor == null || compressedLength >= len) {
		compressMethod = COMPRESSION_METHOD_RAW;
		compressedLength = len;
		System.arraycopy(bytes, off, outputBytes, HEADER_LENGTH, len);
	} else {
		compressMethod = COMPRESSION_METHOD_LZ4;
	}

	outputBytes[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
	writeIntLE(compressedLength, outputBytes, MAGIC_LENGTH + 1);
	writeIntLE(len, outputBytes, MAGIC_LENGTH + 5);
	writeIntLE(check, outputBytes, MAGIC_LENGTH + 9);

	outputBuf.tail(HEADER_LENGTH + compressedLength);

	return outputBuf;
}

Source File: LZ4CompressorTest.java From kylin with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }

Source File: ODAGPartLZ4Wrapper.java From Arabesque with Apache License 2.0

5 votes

private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}

Source File: ODAGStashLZ4Wrapper.java From Arabesque with Apache License 2.0

5 votes

private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}

Source File: LZ4ObjectCache.java From Arabesque with Apache License 2.0

5 votes

private void compressDataOutput() {
    if (!Configuration.get().isUseCompressedCaches() || !uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}

Source File: Lz4CompressUtils.java From hibernate4-memcached with Apache License 2.0

5 votes

public static byte[] compress(final byte[] src) {
    if (src == null) {
        throw new IllegalArgumentException("src must not be null.");
    }

    LZ4Compressor compressor = factory.fastCompressor();

    return compressor.compress(src);
}

Source File: CompressionUtil.java From consulo with Apache License 2.0

5 votes

@Nonnull
public static Object compressStringRawBytes(@Nonnull CharSequence string) {
  int length = string.length();
  if (length < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(length);
    @Nonnull DataOutput out = new DataOutputStream(bytes);

    for (int i = 0; i < length; i++) {
      char c = string.charAt(i);
      DataInputOutputUtil.writeINT(out, c);
    }

    LZ4Compressor compressor = compressor();
    int bytesWritten = bytes.size();
    ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(compressor.maxCompressedLength(bytesWritten) + 10));
    DataInputOutputUtil.writeINT(dest, length);
    DataInputOutputUtil.writeINT(dest, bytesWritten - length);
    compressor.compress(ByteBuffer.wrap(bytes.getInternalBuffer(), 0, bytesWritten), dest);

    return dest.position() < length * 2 ? Arrays.copyOf(dest.array(), dest.position()) : string;
  }
  catch (IOException e) {
    e.printStackTrace();
    return string;
  }
}

Source File: NormalSketch.java From vespa with Apache License 2.0

4 votes

/**
 * Serializes the Sketch.
 *
 * Serialization format
 * ==================
 * Original size:     4 bytes
 * Compressed size:   4 bytes
 * Compressed data:   N * 1 bytes
 *
 * Invariant:
 *      compressed size &lt;= original size
 *
 * Special case:
 *      compressed size == original size =&gt; data is uncompressed
 *
 * @param buf Serializer
 */
@Override
protected void onSerialize(Serializer buf) {
    super.onSerialize(buf);
    buf.putInt(null, data.length);
    try {
        LZ4Compressor c = lz4Factory.highCompressor();
        byte[] compressedData = new byte[data.length];
        int compressedSize = c.compress(data, compressedData);
        serializeDataArray(compressedData, compressedSize, buf);
    } catch (LZ4Exception e) {
        // LZ4Compressor.compress will throw this exception if it is unable to compress
        // into compressedData (when compressed size >= original size)
        serializeDataArray(data, data.length, buf);
    }
}

Java Code Examples for net.jpountz.lz4.LZ4Compressor#compress()