Java Code Examples for com.github.luben.zstd.Zstd#decompress()

@Test
public void testMerge() {
    final String origin1 = "Benedict";
    final String origin2 = "Jin";

    final byte[] originBytes1 = origin1.getBytes(StandardCharsets.UTF_8);
    info(originBytes1);
    final byte[] originBytes2 = origin2.getBytes(StandardCharsets.UTF_8);
    info(originBytes2);

    final byte[] compressedBytes1 = Zstd.compress(originBytes1);
    info(compressedBytes1);
    final byte[] compressedBytes2 = Zstd.compress(originBytes2);
    info(compressedBytes2);

    final byte[] compressedBytes = ByteUtils.merge(compressedBytes1, compressedBytes2);
    info(compressedBytes);

    final long size = Zstd.decompressedSize(compressedBytes);
    System.out.println(size);
    System.out.println(compressedBytes.length);
    final byte[] decompress = Zstd.decompress(compressedBytes, (int) size * 10);
    Assert.assertEquals(origin1 + origin2, new String(decompress, StandardCharsets.UTF_8));
}
 

@Override
public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException {
    ByteBuf uncompressed = PulsarByteBufAllocator.DEFAULT.directBuffer(uncompressedLength, uncompressedLength);

    if (encoded.hasMemoryAddress()) {
        Zstd.decompressUnsafe(uncompressed.memoryAddress(), uncompressedLength,
                encoded.memoryAddress() + encoded.readerIndex(),
                encoded.readableBytes());
    } else {
        ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength);
        ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes());

        Zstd.decompress(uncompressedNio, encodedNio);
    }

    uncompressed.writerIndex(uncompressedLength);
    return uncompressed;
}
 

/**
 * Reads a block of zstd-compressed data. This method
 * expects the following ints to be the compressed size,
 * and uncompressed size respectively.
 *
 * @return the uncompressed data
 * @throws IOException if the bytes cannot be read
 * @throws IllegalArgumentException if the uncompressed length doesn't match
 */
public byte[] readCompressed() throws IOException {
    int compressedLength = readInt();
    int uncompressedLength = readInt();

    byte[] compressed = readByteArray(compressedLength);
    byte[] data = Zstd.decompress(compressed, uncompressedLength);

    if (data.length != uncompressedLength) {
        throw new IllegalArgumentException("Uncompressed length doesn't match");
    }

    return data;
}
 

@Test
public void testZstdCompressingDecompressingStream() throws IOException {

    String sample = randomString(DEFAULT_BLOCK_SIZE);
    ByteBuf source = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    source.writeBytes(sample.getBytes());

    ByteBuf target = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    int idx = target.writerIndex();
    ByteBuffer buffer = target.internalNioBuffer(idx + HEADER_LENGTH, target.writableBytes() - HEADER_LENGTH);

    int compressLength = Zstd.compress(
            buffer,
            source.internalNioBuffer(source.readerIndex(), source.readableBytes()),
            0);
    target.writerIndex(idx + HEADER_LENGTH + compressLength);
    logger.info("{}", target.readerIndex());
    logger.info("{}", target.readableBytes());
    logger.info("{}", target.writerIndex());

    ByteBuf output = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    ByteBuffer outputBuffer = output.internalNioBuffer(output.writerIndex(), output.writableBytes());

    ByteBuffer inputBuffer = target.internalNioBuffer(target.readerIndex() + HEADER_LENGTH, target.readableBytes() - HEADER_LENGTH);

    logger.info("{}", inputBuffer.position());
    logger.info("{}", inputBuffer.limit());
    Zstd.decompress(outputBuffer, inputBuffer);

    outputBuffer.flip();
    output = Unpooled.wrappedBuffer(outputBuffer);

    String val = output.toString(Charset.defaultCharset());

    Assert.assertEquals(sample, val);
}
 

@Test
public void testSimple() {
    final String origin = "yuzhouwan.com, www.yuzhouwan.com, unknown.yuzhouwan.com";

    final byte[] originBytes = origin.getBytes(StandardCharsets.UTF_8);
    info(originBytes);

    final byte[] compressedBytes = Zstd.compress(originBytes);
    info(compressedBytes);

    final int size = compressSize(compressedBytes);
    final byte[] decompress = Zstd.decompress(compressedBytes, size);
    Assert.assertEquals(origin, new String(decompress, StandardCharsets.UTF_8));
}
 

/**
 * Method for obtaining decompressed data for layout 2.
 */
private byte[] getDecompressedBlockData(ReadOnlyGeneticVariantBgen variant) throws IOException {
	// First extend the genetic variant with ids, alleles, etc.
	// This makes sure the file pointer is in the correct location for the next step.
	variant.extendWithAdditionalVariantData();

	// Extract the variant genotype data block info starting from the current position of the
	// file pointer, which should be right after all alleles for a specific variant.
	VariantGenotypeBlockInfo variantGenotypeBlockInfo = extractVariantGenotypeDataBlockInfo();

	long decompressedVariantBlockLength = variantGenotypeBlockInfo.getDecompressedBlockLength();
	long variantBlockLength = variantGenotypeBlockInfo.getBlockLength();
	long variantProbabilitiesStartPosition = variantGenotypeBlockInfo.getVariantProbabilitiesStartPosition();

	if (decompressedVariantBlockLength > Integer.MAX_VALUE - 5) {
		throw new GenotypeDataException(String.format(
				"Length of decompressed genotype data exceeds maximum supported value of (2^31)-6 (%d)",
				decompressedVariantBlockLength));
	}

	if (variantBlockLength > Integer.MAX_VALUE - 5) {
		throw new GenotypeDataException(String.format(
				"Length of compressed genotype data exceeds maximum supported value of (2^31)-6 (%d)", variantBlockLength));
	}

	// Initialize byte arrays.
	byte[] compressedBlockData = new byte[(int) variantBlockLength];
	byte[] decompressedBlockData = new byte[(int) decompressedVariantBlockLength];

	// Read the compressed / uncompressed data starting from the correct location.
	this.bgenFile.seek(variantProbabilitiesStartPosition);
	this.bgenFile.read(compressedBlockData, 0, (int) variantBlockLength);

	switch (snpBlockRepresentation) {

		case compression_1:
			decompressVariantBlockGzip(
					compressedBlockData,
					decompressedBlockData);
			break;

		case compression_2:
			Zstd.decompress(
					decompressedBlockData,
					compressedBlockData);
			break;

		default:
			decompressedBlockData = compressedBlockData;
			break;
	}
	return decompressedBlockData;
}
 

@Test
public void testEncodeWithDefaultBlockSize() {
    String sample = randomString(DEFAULT_BLOCK_SIZE);

    EmbeddedChannel channel = new EmbeddedChannel(encoder);

    ByteBuf byteBuf = Unpooled.directBuffer(DEFAULT_BLOCK_SIZE);
    byte[] sampleBytes = sample.getBytes();
    byteBuf.ensureWritable(sampleBytes.length);
    byteBuf.writeBytes(sampleBytes);

    Assert.assertTrue(channel.writeOutbound(byteBuf));

    ByteBuf target = channel.readOutbound();
    Assert.assertNotNull(target);

    ByteBuffer buffer = ByteBuffer.allocateDirect(sampleBytes.length);
    buffer.clear();

    target.readerIndex(target.readerIndex() + HEADER_LENGTH);
    ByteBuffer source = target.internalNioBuffer(target.readerIndex(), target.readableBytes());

    Zstd.decompress(buffer, source);

    buffer.flip();
    ByteBuf buf = Unpooled.wrappedBuffer(buffer);

    String val = buf.toString(Charset.defaultCharset());

    Assert.assertEquals(sample, val);
}