org.xerial.snappy.SnappyOutputStream Java Examples

public static DataOutputStream getConpressedOutputStream(MessageCompressionType type, OutputStream outputStream) throws IOException {
  switch (type) {
    case NONE:
      return new DataOutputStream(outputStream);
    case SNAPPY:
      return new DataOutputStream(new SnappyOutputStream(outputStream));
    case GZIP:
      try {
        return new DataOutputStream(new GZIPOutputStream(outputStream));
      } catch (IOException e) {
        LOG.error("Create Gzip OuputStream failed", e);
        throw e;
      }
    default:
      throw new RuntimeException("Unsupported Compression type: " + type);
  }
}
 

public static boolean testSnappySupport() {
    try {
        String testString = "Hello World";

        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        SnappyOutputStream out = new SnappyOutputStream(baos);
        out.write(testString.getBytes());
        out.close();
        byte[] bytes = baos.toByteArray();

        ByteArrayInputStream bain = new ByteArrayInputStream(bytes);
        SnappyInputStream in = new SnappyInputStream(bain);
        String result = new String(IOUtils.toBytes(in));
        if (!result.equals(testString)) {
            throw new VertexiumException("uncompressed string did not match compressed string");
        }
        return true;
    } catch (Throwable ex) {
        LOGGER.error("Could not verify support for snappy compression", ex);
        return false;
    }
}
 

@Override
public BlobHolder filterPut(byte[] valueToFilter, String tag) {
  if (valueToFilter.length < filterSize) {
    return new Tagged(valueToFilter, tag);
  }
  final Path file = Paths.get(UUID.randomUUID().toString() + ".blob");
  final Path path = base.resolve(file);

  logger.debug("Value size {} bytes larger than limit {} bytes - storing actual data at {}",
    valueToFilter.length, filterSize, path);

  try (final SnappyOutputStream snappyOutputStream = new SnappyOutputStream(Files.newOutputStream(path))){
    snappyOutputStream.write(valueToFilter);
    return new Blob(file, tag);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void save(String saveFileName) throws IOException {
  Kryo kryo = CubeUtils.getKryoWithRegistrations();
  OutputStream zip = new SnappyOutputStream(new FileOutputStream(saveFileName));
  Output output = new Output(zip);
  kryo.writeClassAndObject(output, partitions);
  // zip.closeEntry();
  output.close();
  // zip.close();
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

/**
 * Convert a Configuration to compressed ByteString using Protocol buffer
 *
 * @param conf
 *          : Configuration to be converted
 * @return PB ByteString (compressed)
 * @throws java.io.IOException
 */
public static ByteString createByteStringFromConf(Configuration conf) throws IOException {
  Objects.requireNonNull(conf, "Configuration must be specified");
  ByteString.Output os = ByteString.newOutput();
  SnappyOutputStream compressOs = new SnappyOutputStream(os);
  try {
    writeConfInPB(compressOs, conf);
  } finally {
    if (compressOs != null) {
      compressOs.close();
    }
  }
  return os.toByteString();
}
 

/**
 * Serializes the argument into an array of bytes, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when serializing
 */
public static byte[] serializeToCompressedByteArray(Object value) {
    try {
        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        try (ObjectOutputStream oos = new ObjectOutputStream(new SnappyOutputStream(buffer))) {
            oos.writeObject(value);
        }
        return buffer.toByteArray();
    } catch (IOException exn) {
        throw new IllegalArgumentException(
            "unable to serialize " + value,
            exn);
    }
}
 

/**
 * Serializes the argument into an array of bytes, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when serializing
 */
public static byte[] serializeToCompressedByteArray(Object value) {
    try {
        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
        try (ObjectOutputStream oos = new ObjectOutputStream(new SnappyOutputStream(buffer))) {
            oos.writeObject(value);
        }
        return buffer.toByteArray();
    } catch (IOException exn) {
        throw new IllegalArgumentException(
            "unable to serialize " + value,
            exn);
    }
}
 

private OutputStream getCompressedStream(final OutputStream outputStream) throws IOException {
    Objects.requireNonNull(outputStream, "outputStream cannot be null");

    switch (config.getCompressionType()) {
        case ZSTD:
            return new ZstdOutputStream(outputStream);
        case GZIP:
            return new GZIPOutputStream(outputStream);
        case SNAPPY:
            return new SnappyOutputStream(outputStream);
        default:
            return outputStream;
    }
}
 

/**
 * Serializes the argument into an array of bytes, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when serializing
 */
public static byte[] serializeToByteArray(Serializable value) {
  try {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    try (ObjectOutputStream oos = new ObjectOutputStream(new SnappyOutputStream(buffer))) {
      oos.writeObject(value);
    }
    return buffer.toByteArray();
  } catch (IOException exn) {
    throw new IllegalArgumentException("unable to serialize " + value, exn);
  }
}
 

private OutputStream wrapIfNeeded(OutputStream o) throws IOException {
  switch (splitCompression) {
    case UNCOMPRESSED:
      return o;
    case SNAPPY:
      return new SnappyOutputStream(o);
    default:
      throw new IllegalArgumentException(String.format("Unsupported compression type: %s", splitCompression));
  }
}
 

public static OutputStream apply(KafkaCompressionCodec codec, OutputStream stream, byte messageVersion) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPOutputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyOutputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockOutputStream(stream);
        default:
            throw new RuntimeException(String.format("unknown codec: %s", codec));
    }
}
 

public static OutputStream apply(KafkaCompressionCodec codec, OutputStream stream, byte messageMagic) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPOutputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyOutputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockOutputStream(stream);
        default:
            throw new UnknownCodecException(String.format("unknown codec: %s", codec));
    }
}
 

public static OutputStream apply(KafkaCompressionCodec codec, OutputStream stream, byte messageVersion) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPOutputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyOutputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockOutputStream(stream);
        default:
            throw new RuntimeException(String.format("unknown codec: %s", codec));
    }
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

@Override
protected OutputStream getCompressOutputStream(OutputStream outputStream) {
    return new SnappyOutputStream(outputStream);
}
 

@Override
public OutputStream compress(OutputStream output) {
  return new SnappyOutputStream(output);
}
 

public SnappyCompression(InputStream inputStream, OutputStream outputStream) throws IOException {
	super(inputStream != null ? new SnappyInputStream(inputStream) : null, //
               outputStream != null ? new SnappyOutputStream(outputStream) : null);
}
 

@Override
public void close() throws IOException {
  SnappyOutputStream zos = (SnappyOutputStream) delegate;
  zos.flush();
  zos.close();
}
 

@Override
public void close() throws IOException {
  SnappyOutputStream zos = (SnappyOutputStream) delegate;
  zos.flush();
  zos.close();
}