Java Code Examples for com.github.luben.zstd.Zstd#compress()

@Test
public void testMerge() {
    final String origin1 = "Benedict";
    final String origin2 = "Jin";

    final byte[] originBytes1 = origin1.getBytes(StandardCharsets.UTF_8);
    info(originBytes1);
    final byte[] originBytes2 = origin2.getBytes(StandardCharsets.UTF_8);
    info(originBytes2);

    final byte[] compressedBytes1 = Zstd.compress(originBytes1);
    info(compressedBytes1);
    final byte[] compressedBytes2 = Zstd.compress(originBytes2);
    info(compressedBytes2);

    final byte[] compressedBytes = ByteUtils.merge(compressedBytes1, compressedBytes2);
    info(compressedBytes);

    final long size = Zstd.decompressedSize(compressedBytes);
    System.out.println(size);
    System.out.println(compressedBytes.length);
    final byte[] decompress = Zstd.decompress(compressedBytes, (int) size * 10);
    Assert.assertEquals(origin1 + origin2, new String(decompress, StandardCharsets.UTF_8));
}
 

/**
 * Write the genotype probability data from the byte buffer in compressed form to the output channel.
 *
 * @param bgenOutputByteChannel The channel that is able to pass byte buffers to the output stream.
 * @param genotypeDataBlockBuffer The byte buffer that holds the genotype probability data.
 * @throws IOException If an I/O exception has occurred.
 * @return The number of bytes written.
 */
private long writeCompressedBgenGenotypeDataBlock(WritableByteChannel bgenOutputByteChannel,
												  ByteBuffer genotypeDataBlockBuffer) throws IOException {

	// Compress the genotype data using the Zstandard compression method.
	// Compression level 3 is used since this was the defualt when writing this.
	// (The byte buffer compression method does not have have an overloaded method with default compression level)
	ByteBuffer compressedGenotypeDataBuffer = Zstd.compress(genotypeDataBlockBuffer, 3);
	// If this was successful, the limit is equal to the position of the original buffer.
	// Check if the alternative condition applies.
	if (genotypeDataBlockBuffer.position() != genotypeDataBlockBuffer.limit()) {
		throw new GenotypeDataException("Zstd compression of genotype data failed, exiting");
	}

	// Get the length of the compressed variant probability data.
	int lengthOfVariantProbabilityData =
			compressedGenotypeDataBuffer.limit() + 4; // Add 4 since the D field takes 4 bytes
	// Get the decompressed length of variant probability data.
	int decompressedLengthOfVariantProbabilityData = genotypeDataBlockBuffer.capacity();

	// Write the (decompressed) length to a bytebuffer.
	ByteBuffer genotypeBlockLengths = ByteBuffer.allocate(8)
			.order(ByteOrder.LITTLE_ENDIAN);

	genotypeBlockLengths.putInt(lengthOfVariantProbabilityData);
	genotypeBlockLengths.putInt(decompressedLengthOfVariantProbabilityData);

	genotypeBlockLengths.flip(); // reset pointer

	// First write the lengths.
	int numberOfBytesWritten = bgenOutputByteChannel.write(genotypeBlockLengths);
	// Now write the actual data.
	return numberOfBytesWritten + bgenOutputByteChannel.write(compressedGenotypeDataBuffer);
}
 

@Override
public ByteBuf encode(ByteBuf source) {
    int uncompressedLength = source.readableBytes();
    int maxLength = (int) Zstd.compressBound(uncompressedLength);

    ByteBuf target = PulsarByteBufAllocator.DEFAULT.directBuffer(maxLength, maxLength);
    int compressedLength;

    if (source.hasMemoryAddress()) {
        compressedLength = (int) Zstd.compressUnsafe(target.memoryAddress(), maxLength,
                source.memoryAddress() + source.readerIndex(),
                uncompressedLength, ZSTD_COMPRESSION_LEVEL);
    } else {
        ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes());
        ByteBuffer targetNio = target.nioBuffer(0, maxLength);

        compressedLength = Zstd.compress(targetNio, sourceNio, ZSTD_COMPRESSION_LEVEL);
    }

    target.writerIndex(compressedLength);
    return target;
}
 

private void flushBufferedData(ByteBuf out) {
    final int flushableBytes = buffer.readableBytes();
    if (flushableBytes == 0) {
        return;
    }
    checksum.reset();
    checksum.update(buffer.internalNioBuffer(buffer.readerIndex(), flushableBytes));
    final int check = (int) checksum.getValue();

    final int bufSize = (int) Zstd.compressBound(flushableBytes) + HEADER_LENGTH;
    out.ensureWritable(bufSize);
    final int idx = out.writerIndex();
    int compressedLength;
    try {
        ByteBuffer outNioBuffer = out.internalNioBuffer(idx + HEADER_LENGTH, out.writableBytes() - HEADER_LENGTH);
        compressedLength = Zstd.compress(
                outNioBuffer,
                buffer.internalNioBuffer(buffer.readerIndex(), flushableBytes),
                DEFAULT_COMPRESS_LEVEL);
    } catch (Exception e) {
        throw new CompressionException(e);
    }
    final int blockType;
    if (compressedLength >= flushableBytes) {
        blockType = BLOCK_TYPE_NON_COMPRESSED;
        compressedLength = flushableBytes;
        out.setBytes(idx + HEADER_LENGTH, buffer, 0, flushableBytes);
    } else {
        blockType = BLOCK_TYPE_COMPRESSED;
    }

    out.setInt(idx, MAGIC_NUMBER);
    out.setByte(idx + TOKEN_OFFSET, (byte) (blockType | compressionLevel));
    out.setIntLE(idx + COMPRESSED_LENGTH_OFFSET, compressedLength);
    out.setIntLE(idx + DECOMPRESSED_LENGTH_OFFSET, flushableBytes);
    out.setIntLE(idx + CHECKSUM_OFFSET, check);
    out.writerIndex(idx + HEADER_LENGTH + compressedLength);
    buffer.clear();
}
 

@Test
public void testZstdCompressingDecompressingStream() throws IOException {

    String sample = randomString(DEFAULT_BLOCK_SIZE);
    ByteBuf source = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    source.writeBytes(sample.getBytes());

    ByteBuf target = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    int idx = target.writerIndex();
    ByteBuffer buffer = target.internalNioBuffer(idx + HEADER_LENGTH, target.writableBytes() - HEADER_LENGTH);

    int compressLength = Zstd.compress(
            buffer,
            source.internalNioBuffer(source.readerIndex(), source.readableBytes()),
            0);
    target.writerIndex(idx + HEADER_LENGTH + compressLength);
    logger.info("{}", target.readerIndex());
    logger.info("{}", target.readableBytes());
    logger.info("{}", target.writerIndex());

    ByteBuf output = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    ByteBuffer outputBuffer = output.internalNioBuffer(output.writerIndex(), output.writableBytes());

    ByteBuffer inputBuffer = target.internalNioBuffer(target.readerIndex() + HEADER_LENGTH, target.readableBytes() - HEADER_LENGTH);

    logger.info("{}", inputBuffer.position());
    logger.info("{}", inputBuffer.limit());
    Zstd.decompress(outputBuffer, inputBuffer);

    outputBuffer.flip();
    output = Unpooled.wrappedBuffer(outputBuffer);

    String val = output.toString(Charset.defaultCharset());

    Assert.assertEquals(sample, val);
}
 

/**
 * @param compactPage Compacted page.
 * @param compactSize Compacted page size.
 * @param compressLevel Compression level.
 * @return Compressed page.
 */
private ByteBuffer compressPageZstd(ByteBuffer compactPage, int compactSize, int compressLevel) {
    ByteBuffer compressedPage = compressBuf.get();

    copyPageHeader(compactPage, compressedPage, compactSize);
    Zstd.compress(compressedPage, compactPage, compressLevel);

    compactPage.flip();
    compressedPage.flip();

    return compressedPage;
}
 

@Test
public void testSimple() {
    final String origin = "yuzhouwan.com, www.yuzhouwan.com, unknown.yuzhouwan.com";

    final byte[] originBytes = origin.getBytes(StandardCharsets.UTF_8);
    info(originBytes);

    final byte[] compressedBytes = Zstd.compress(originBytes);
    info(compressedBytes);

    final int size = compressSize(compressedBytes);
    final byte[] decompress = Zstd.decompress(compressedBytes, size);
    Assert.assertEquals(origin, new String(decompress, StandardCharsets.UTF_8));
}
 

@Test
public void testMixed() {
    final String origin1 = "compress";
    final String origin2 = "origin";

    final byte[] originBytes1 = origin1.getBytes(StandardCharsets.UTF_8);
    info(originBytes1);
    final byte[] originBytes2 = origin2.getBytes(StandardCharsets.UTF_8);
    info(originBytes2);

    final byte[] compressedBytes1 = Zstd.compress(originBytes1);
    info(compressedBytes1);

    final byte[] fixedBytes = ByteUtils.merge(compressedBytes1, originBytes2);
    info(fixedBytes);

    final long size = Zstd.decompressedSize(fixedBytes);
    System.out.println(size);
    System.out.println(fixedBytes.length);

    final ZstdException e = Assert.assertThrows(ZstdException.class, () -> {
        final byte[] decompress = Zstd.decompress(fixedBytes, compressSize(fixedBytes));
        System.out.println(new String(decompress, StandardCharsets.UTF_8));
    });
    Assert.assertEquals("Src size is incorrect", e.getMessage());
    System.err.println("Compressed and uncompressed arrays should not be mixed together.");
}