Java Code Examples for org.xerial.snappy.Snappy#compress()

public static byte[] compress(byte[] uncompressData) {
    try {
        switch (CMP) {
            case SNAPPY:
                return Snappy.compress(uncompressData);
            case GZIP:
                ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
                GZIPOutputStream gzout = new GZIPOutputStream(out);
                gzout.write(uncompressData, 0, uncompressData.length);
                gzout.close();
                return out.toByteArray();
            default:
                throw new RuntimeException("Should not happen!");
        }
    } catch (Exception e) {
        throw new IllegalStateException(e);
    }
}
 

@Override
public ByteBuf encode(ByteBuf source) {
    int uncompressedLength = source.readableBytes();
    int maxLength = Snappy.maxCompressedLength(uncompressedLength);

    ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes());

    ByteBuf target = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength);
    ByteBuffer targetNio = target.nioBuffer(0, maxLength);

    int compressedLength = 0;
    try {
        compressedLength = Snappy.compress(sourceNio, targetNio);
    } catch (IOException e) {
        log.error("Failed to compress to Snappy: {}", e.getMessage());
    }
    target.writerIndex(compressedLength);
    return target;
}
 

public Frame compress(Frame frame) throws IOException
{
    byte[] input = CBUtil.readRawBytes(frame.body);
    ByteBuf output = CBUtil.allocator.heapBuffer(Snappy.maxCompressedLength(input.length));

    try
    {
        int written = Snappy.compress(input, 0, input.length, output.array(), output.arrayOffset());
        output.writerIndex(written);
    }
    catch (final Throwable e)
    {
        output.release();
        throw e;
    }
    finally
    {
        //release the old frame
        frame.release();
    }

    return frame.with(output);
}
 

/**
 * Fills specified buffer with compressed data. Returns actual number
 * of bytes of compressed data. A return value of 0 indicates that
 * needsInput() should be called in order to determine if more input
 * data is required.
 *
 * @param buffer   Buffer for the compressed data
 * @param off Start offset of the data
 * @param len Size of the buffer
 * @return The actual number of bytes of compressed data.
 */
@Override
public synchronized int compress(byte[] buffer, int off, int len) throws IOException {
  SnappyUtil.validateBuffer(buffer, off, len);

  if (needsInput()) {
    // No buffered output bytes and no input to consume, need more input
    return 0;
  }

  if (!outputBuffer.hasRemaining()) {
    // There is uncompressed input, compress it now
    int maxOutputSize = Snappy.maxCompressedLength(inputBuffer.position());
    if (maxOutputSize > outputBuffer.capacity()) {
      ByteBuffer oldBuffer = outputBuffer;
      outputBuffer = ByteBuffer.allocateDirect(maxOutputSize);
      CleanUtil.cleanDirectBuffer(oldBuffer);
    }
    // Reset the previous outputBuffer
    outputBuffer.clear();
    inputBuffer.limit(inputBuffer.position());
    inputBuffer.position(0);

    int size = Snappy.compress(inputBuffer, outputBuffer);
    outputBuffer.limit(size);
    inputBuffer.limit(0);
    inputBuffer.rewind();
  }

  // Return compressed output up to 'len'
  int numBytes = Math.min(len, outputBuffer.remaining());
  outputBuffer.get(buffer, off, numBytes);    
  bytesWritten += numBytes;
  return numBytes;	    
}
 

public static byte[] compressFilter(byte[] datas) {
    if (Constants.COMPRESS_FILE) {
        try {
            datas = Snappy.compress(datas);
        } catch (IOException e) {
            log.error("Compress error!", e);
        }
    }
    return datas;
}
 

@Override
public byte[] compress(byte[] input) {
  try {
    return Snappy.compress(input);
  } catch (IOException e) {
    throw new CompressionException(e);
  }
}
 

private static void serializeDoubleArray(final DataOutput out, final double[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(DOUBLE_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(DOUBLE_ARRAY);
    LongPacker.packInt(out, val.length);
    for (double s : val) {
      out.writeDouble(s);
    }
  }
}
 

private static void serializeCharArray(final DataOutput out, final char[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(CHAR_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(CHAR_ARRAY);
    LongPacker.packInt(out, val.length);
    for (char s : val) {
      out.writeChar(s);
    }
  }
}
 

@Override
public ByteBuf encode(Object in) throws IOException {
    ByteBuf encoded = innerCodec.getValueEncoder().encode(in);
    byte[] bytes = new byte[encoded.readableBytes()];
    encoded.readBytes(bytes);
    encoded.release();
    byte[] res = Snappy.compress(bytes);
    return Unpooled.wrappedBuffer(res);
}
 

public byte[] compress(final byte[] uncompressed) {
  checkNotNull(uncompressed, "input data must not be null");
  try {
    return Snappy.compress(uncompressed);
  } catch (final IOException e) {
    throw new FramingException("Snappy compression failed", e);
  }
}
 

public static int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) throws IOException {
    if (installed) {
        return Snappy.compress(input, inputOffset, inputLength, output, outputOffset);
    }
    System.arraycopy(input, inputOffset, output, outputOffset, inputLength);
    return inputLength;
}
 

private static void serializeShortArray(final DataOutput out, final short[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(SHORT_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(SHORT_ARRAY);
    LongPacker.packInt(out, val.length);
    for (short s : val) {
      out.writeShort(s);
    }
  }
}
 

@Override
public void flush() throws IOException {
  if (pending == 0) {
    return;
  }

  int compressedSize = Snappy.compress(uncompressedBuffer, 0, pending, compressedBuffer, 0);
  Util.writeUnsignedVLQ(compressedSize, output);
  output.write(compressedBuffer, 0, compressedSize);
  output.flush();
  pending = 0;
  listener.afterFlush();
}
 

private void writeCompressedBuf(ArrowBuf buf, OutputStream output) throws IOException {
  long rawLength = buf.readableBytes();
  for (long posn = 0; posn < rawLength; posn += RAW_CHUNK_SIZE_TO_COMPRESS) {
    /* we compress 32KB chunks at a time; the last chunk might be smaller than 32KB */
    int lengthToCompress = (int) Math.min(RAW_CHUNK_SIZE_TO_COMPRESS, rawLength - posn);

    /* allocate direct buffers to hold raw and compressed data */
    ByteBuffer rawDirectBuffer = buf.nioBuffer(posn, lengthToCompress);
    /* Since we don't know the exact size of compressed data, we can
     * allocate the compressed buffer of same size as raw data. However,
     * there could be cases where Snappy does not compress the data and the
     * compressed stream is of size larger (raw data + compression metadata)
     * than raw data. To handle these cases, we allocate compressed buffer
     * slightly larger than raw buffer. If we don't do this, Snappy.compress
     * will segfault.
     */
    final int maxCompressedLength = Snappy.maxCompressedLength(lengthToCompress);
    try (ArrowBuf cBuf = allocator.buffer(maxCompressedLength)) {
      ByteBuffer compressedDirectBuffer = cBuf.nioBuffer(0, maxCompressedLength);
      rawDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);
      compressedDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);

      /* compress */
      int compressedLength = Snappy.compress(rawDirectBuffer, compressedDirectBuffer);

      /* get compressed data into byte array for serializing to output stream */
      /* Use current thread buffer (safe to do since I/O operation is blocking) */
      final byte[] tmpBuffer = REUSABLE_LARGE_BUFFER.get();
      compressedDirectBuffer.get(tmpBuffer, 0, compressedLength);

      /* serialize the length of compressed data */
      output.write(getByteArrayFromLEInt(REUSABLE_SMALL_BUFFER.get(), compressedLength));
      /* serialize the compressed data */
      output.write(tmpBuffer, 0, compressedLength);
    }
  }
}
 

@Override
public int compress(ByteBuffer data, ByteBuffer compressed) throws IOException {
  return Snappy.compress(data, compressed);
}
 

@Override
public byte[] serialize(Object obj) throws IOException {
	return Snappy.compress(inner.serialize(obj));
}
 

@Override
public int compress(ByteBuffer inDecompressed, ByteBuffer outCompressed)
    throws IOException {
  return Snappy.compress(inDecompressed, outCompressed);
}
 

static byte[] snappyCompressData(byte[] data) throws IOException {
    return Snappy.compress(data);
}
 

@Test
public void TestSnappyStream() throws IOException {
  SnappyCodec codec = new SnappyCodec();
  codec.setConf(new Configuration());
  
  int blockSize = 1024;
  int inputSize = blockSize * 1024;
 
  byte[] input = new byte[inputSize];
  for (int i = 0; i < inputSize; ++i) {
    input[i] = (byte)i;
  }

  ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
  
  CompressionOutputStream compressor = codec.createOutputStream(compressedStream);
  int bytesCompressed = 0;
  while (bytesCompressed < inputSize) {
    int len = Math.min(inputSize - bytesCompressed, blockSize);
    compressor.write(input, bytesCompressed, len);
    bytesCompressed += len;
  }
  compressor.finish();
  
  byte[] rawCompressed = Snappy.compress(input);
  byte[] codecCompressed = compressedStream.toByteArray();
  
  // Validate that the result from the codec is the same as if we compressed the 
  // buffer directly.
  assertArrayEquals(rawCompressed, codecCompressed);

  ByteArrayInputStream inputStream = new ByteArrayInputStream(codecCompressed);    
  CompressionInputStream decompressor = codec.createInputStream(inputStream);
  byte[] codecDecompressed = new byte[inputSize];
  int bytesDecompressed = 0;
  int numBytes;
  while ((numBytes = decompressor.read(codecDecompressed, bytesDecompressed, blockSize)) != 0) {
    bytesDecompressed += numBytes;
    if (bytesDecompressed == inputSize) break;
  }
  
  byte[] rawDecompressed = Snappy.uncompress(rawCompressed);
  
  assertArrayEquals(input, rawDecompressed);
  assertArrayEquals(input, codecDecompressed);
}
 

@Override
public byte[] compress(byte[] data) throws IOException {
	return Snappy.compress(data);
}