org.xerial.snappy.SnappyInputStream Java Examples

private InputStream getDecompressedStream(final InputStream inputStream, final String compression)
        throws IOException {
    Objects.requireNonNull(inputStream, "inputStream cannot be null");
    Objects.requireNonNull(compression, "compression cannot be null");

    final CompressionType compressionType = CompressionType.forName(compression);
    switch (compressionType) {
        case ZSTD:
            return new ZstdInputStream(inputStream);
        case GZIP:
            return new GZIPInputStream(inputStream);
        case SNAPPY:
            return new SnappyInputStream(inputStream);
        default:
            return inputStream;
    }
}
 

public static boolean testSnappySupport() {
    try {
        String testString = "Hello World";

        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        SnappyOutputStream out = new SnappyOutputStream(baos);
        out.write(testString.getBytes());
        out.close();
        byte[] bytes = baos.toByteArray();

        ByteArrayInputStream bain = new ByteArrayInputStream(bytes);
        SnappyInputStream in = new SnappyInputStream(bain);
        String result = new String(IOUtils.toBytes(in));
        if (!result.equals(testString)) {
            throw new VertexiumException("uncompressed string did not match compressed string");
        }
        return true;
    } catch (Throwable ex) {
        LOGGER.error("Could not verify support for snappy compression", ex);
        return false;
    }
}
 

public static DataInputStream decodeHeader(InputStream in, byte expectedTypeId) throws IOException {
    byte[] header = new byte[ElementData.HEADER.length];
    int read = in.read(header);
    if (read != header.length) {
        throw new IOException("Unexpected header length. Expected " + ElementData.HEADER.length + " found " + read);
    }
    if (Arrays.equals(header, ElementData.SNAPPY_HEADER)) {
        return decodeHeader(new SnappyInputStream(in), expectedTypeId);
    }
    if (!Arrays.equals(header, ElementData.HEADER)) {
        throw new IOException("Unexpected header");
    }
    int typeId = in.read();
    if (typeId != expectedTypeId) {
        throw new IOException("Unexpected type id. Expected " + expectedTypeId + " found " + typeId);
    }
    return new DataInputStream(in);
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

@Test
public void testCreateInputStream() throws IOException {
  SnappyCompressionProvider provider =
    (SnappyCompressionProvider) factory.getCompressionProviderByName( PROVIDER_NAME );
  SnappyInputStream in = createSnappyInputStream();
  SnappyCompressionInputStream inStream = new SnappyCompressionInputStream( in, provider );
  assertNotNull( inStream );
  SnappyCompressionInputStream ncis = provider.createInputStream( in );
  assertNotNull( ncis );
}
 

public static void addToConfFromByteString(Configuration configuration, ByteString byteString)
    throws IOException {
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
  }
}
 

public static Configuration createConfFromBaseConfAndPayload(TaskContext context)
    throws IOException {
  Configuration baseConf = context.getContainerConfiguration();
  Configuration configuration = new Configuration(baseConf);
  UserPayload payload = context.getUserPayload();
  ByteString byteString = ByteString.copyFrom(payload.getPayload());
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) {
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(uncompressIs);
    readConfFromPB(confProto, configuration);
    return configuration;
  }
}
 

/**
 * Convert a byte string to a Configuration object
 *
 * @param byteString byteString representation of the conf created using {@link
 *                   #createByteStringFromConf(org.apache.hadoop.conf.Configuration)}
 * @return Configuration
 * @throws java.io.IOException
 */
public static Configuration createConfFromByteString(ByteString byteString) throws IOException {
  Objects.requireNonNull(byteString, "ByteString must be specified");
  try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) {
    CodedInputStream in = CodedInputStream.newInstance(uncompressIs);
    in.setSizeLimit(Integer.MAX_VALUE);
    DAGProtos.ConfigurationProto confProto = DAGProtos.ConfigurationProto.parseFrom(in);
    Configuration conf = new Configuration(false);
    readConfFromPB(confProto, conf);
    return conf;
  }
}
 

private List<Datum> readInData() throws IOException {
    File f = new File(fileDir + "/" + convertFileName(timeColumn,
                                                      attributes,
                                                      metrics,
                                                      conf.getString(MacroBaseConf.BASE_QUERY,
                                                                     conf.getString(MacroBaseConf.QUERY_NAME,
                                                                                    "cachedQuery"))));
    if (!f.exists()) {
        log.info("Data did not exist; going to read from SQL.");
        return null;
    }

    log.info("On-disk cache exists; loading...");
    InputStream inputStream = new SnappyInputStream(new BufferedInputStream(new FileInputStream(f), 16384));

    Kryo kryo = new Kryo();
    Input input = new Input(inputStream);

    DatumEncoder cachedEncoder = kryo.readObject(input, DatumEncoder.class);

    Integer numBatches = kryo.readObject(input, Integer.class);
    List<Datum> output = null;

    for (int i = 0; i < numBatches; ++i) {
        List<Datum> fromDisk = (List<Datum>) kryo.readClassAndObject(input);
        if (output == null) {
            output = fromDisk;
        } else {
            output.addAll(fromDisk);
        }
    }

    log.info("...loaded!");

    conf.getEncoder().copy(cachedEncoder);
    return output;
}
 

/**
 * Deserializes an object from the given array of bytes, e.g., as
 * serialized using {@link #serializeToCompressedByteArray}, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when
 *                                  deserializing, using the provided description to identify what
 *                                  was being deserialized
 */
public static Object deserializeFromCompressedByteArray(byte[] encodedValue,
                                                        String description) {
    try {
        try (ObjectInputStream ois = new ObjectInputStream(
            new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
            return ois.readObject();
        }
    } catch (IOException | ClassNotFoundException exn) {
        throw new IllegalArgumentException(
            "unable to deserialize " + description,
            exn);
    }
}
 

/**
 * Deserializes an object from the given array of bytes, e.g., as
 * serialized using {@link #serializeToCompressedByteArray}, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when
 *                                  deserializing, using the provided description to identify what
 *                                  was being deserialized
 */
public static Object deserializeFromCompressedByteArray(byte[] encodedValue,
                                                        String description) {
    try {
        try (ObjectInputStream ois = new ObjectInputStream(
            new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
            return ois.readObject();
        }
    } catch (IOException | ClassNotFoundException exn) {
        throw new IllegalArgumentException(
            "unable to deserialize " + description,
            exn);
    }
}
 

/**
 * Deserializes an object from the given array of bytes, e.g., as serialized using {@link
 * #serializeToByteArray}, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when deserializing, using the provided
 *     description to identify what was being deserialized
 */
public static Object deserializeFromByteArray(byte[] encodedValue, String description) {
  try {
    try (ObjectInputStream ois =
        new ContextualObjectInputStream(
            new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
      return ois.readObject();
    }
  } catch (IOException | ClassNotFoundException exn) {
    throw new IllegalArgumentException("unable to deserialize " + description, exn);
  }
}
 

public static InputStream toInputStream(com.google.protobuf.ByteString json, FragmentCodec codec) throws IOException {
  final FragmentCodec c = codec != null ? codec : FragmentCodec.NONE;

  final InputStream input = json.newInput();
  switch(c) {
  case NONE:
    return input;

  case SNAPPY:
    return new SnappyInputStream(input);

  default:
    throw new UnsupportedOperationException("Do not know how to uncompress using " + c + " algorithm.");
  }
}
 

@Override
protected InputStream getDecompressInputStream(InputStream inputStream) {
    try {
        return new SnappyInputStream(inputStream);
    } catch (IOException e) {
        logger.error("[getDecompressInputStream]", e);
    }
    throw new IllegalArgumentException("");
}
 

public static InputStream apply(KafkaCompressionCodec codec, InputStream stream, byte messageVersion) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPInputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyInputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockInputStream(stream, messageVersion == 0); // RecordBatch.MAGIC_VALUE_V0
        default:
            throw new RuntimeException(String.format("unknown codec: %s", codec));
    }
}
 

public static InputStream apply(KafkaCompressionCodec codec, InputStream stream, byte messageMagic) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPInputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyInputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockInputStream(stream, messageMagic == 0); // RecordBatch.MAGIC_VALUE_V0
        default:
            throw new UnknownCodecException(String.format("unknown codec: %s", codec));
    }
}
 

@Test
public void testCreateInputStream() throws IOException {
  SnappyCompressionProvider provider =
    (SnappyCompressionProvider) factory.getCompressionProviderByName( PROVIDER_NAME );
  SnappyInputStream in = createSnappyInputStream();
  SnappyCompressionInputStream inStream = new SnappyCompressionInputStream( in, provider );
  assertNotNull( inStream );
  SnappyCompressionInputStream ncis = provider.createInputStream( in );
  assertNotNull( ncis );
}
 

public static InputStream apply(KafkaCompressionCodec codec, InputStream stream, byte messageVersion) throws IOException {
    switch (codec) {
        case GZIPCompressionCodec:
            return new GZIPInputStream(stream);
        case SnappyCompressionCodec:
            return new SnappyInputStream(stream);
        case LZ4CompressionCodec:
            return new KafkaLZ4BlockInputStream(stream, messageVersion == 0); // RecordBatch.MAGIC_VALUE_V0
        default:
            throw new RuntimeException(String.format("unknown codec: %s", codec));
    }
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}
 

@Override
public void close() throws IOException {
  ( (SnappyInputStream) delegate ).close();
}
 

@Override
public int read() throws IOException {
  return ( (SnappyInputStream) delegate ).read();
}
 

@Override
public int read() throws IOException {
  return ( (SnappyInputStream) delegate ).read();
}
 

@Override
public void close() throws IOException {
  ( (SnappyInputStream) delegate ).close();
}
 

public SnappyCompression(InputStream inputStream, OutputStream outputStream) throws IOException {
	super(inputStream != null ? new SnappyInputStream(inputStream) : null, //
               outputStream != null ? new SnappyOutputStream(outputStream) : null);
}
 

@Test
public void testJSONCompression() throws InitializationException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(ConvertRecord.class);
    final JsonTreeReader jsonReader = new JsonTreeReader();
    runner.addControllerService("reader", jsonReader);

    final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestConvertRecord/schema/person.avsc")));
    final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestConvertRecord/schema/person.avsc")));

    runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
    runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText);
    runner.enableControllerService(jsonReader);

    final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
    runner.addControllerService("writer", jsonWriter);
    runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
    runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
    runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
    runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
    runner.setProperty(jsonWriter, "compression-format", "snappy");
    runner.enableControllerService(jsonWriter);

    runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person.json"));

    runner.setProperty(ConvertRecord.RECORD_READER, "reader");
    runner.setProperty(ConvertRecord.RECORD_WRITER, "writer");

    runner.run();
    runner.assertAllFlowFilesTransferred(ConvertRecord.REL_SUCCESS, 1);

    MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConvertRecord.REL_SUCCESS).get(0);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    try (final SnappyInputStream sis = new SnappyInputStream(new ByteArrayInputStream(flowFile.toByteArray())); final OutputStream out = baos) {
        final byte[] buffer = new byte[8192]; int len;
        while ((len = sis.read(buffer)) > 0) {
            out.write(buffer, 0, len);
        }
        out.flush();
    }

    assertEquals(new String(Files.readAllBytes(Paths.get("src/test/resources/TestConvertRecord/input/person.json"))), baos.toString(StandardCharsets.UTF_8.name()));
}
 

private void receiveMessages() throws IOException
{
    // handshake (true) endpoint versions
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    out.writeInt(MessagingService.current_version);
    out.flush();
    DataInputStream in = new DataInputStream(socket.getInputStream());
    int maxVersion = in.readInt();

    from = CompactEndpointSerializationHelper.deserialize(in);
    // record the (true) version of the endpoint
    MessagingService.instance().setVersion(from, maxVersion);
    logger.debug("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));

    if (compressed)
    {
        logger.debug("Upgrading incoming connection to be compressed");
        if (version < MessagingService.VERSION_21)
        {
            in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
        }
        else
        {
            LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
            Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
            in = new DataInputStream(new LZ4BlockInputStream(socket.getInputStream(),
                                                             decompressor,
                                                             checksum));
        }
    }
    else
    {
        in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), BUFFER_SIZE));
    }

    if (version > MessagingService.current_version)
    {
        // save the endpoint so gossip will reconnect to it
        Gossiper.instance.addSavedEndpoint(from);
        logger.info("Received messages from newer protocol version {}. Ignoring", version);
        return;
    }
    // outbound side will reconnect if necessary to upgrade version

    while (true)
    {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}
 

@Override
public InputStream decompress(InputStream input) throws IOException {
  return new SnappyInputStream(input);
}