Java Code Examples for org.apache.avro.file.CodecFactory#nullCodec()

The following examples show how to use org.apache.avro.file.CodecFactory#nullCodec() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvroRecordWriterTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up 
@Test
public void typical() throws Exception {
  Schema schema = SchemaBuilder
      .builder()
      .record("record")
      .fields()
      .requiredLong("id")
      .requiredString("name")
      .endRecord();
  Record value = new GenericRecordBuilder(schema).set("id", 1L).set("name", "hello").build();
  ByteArrayOutputStream output = new ByteArrayOutputStream();

  Factory factory = new Factory(CodecFactory.nullCodec());
  RecordWriter writer = factory.create(schema, output);
  writer.write(value);
  writer.close();

  SeekableInput input = new SeekableByteArrayInput(output.toByteArray());
  DatumReader<Record> datumReader = new GenericDatumReader<>(schema);
  DataFileReader<Record> dataFileReader = new DataFileReader<>(input, datumReader);
  assertThat(dataFileReader.next(), is(value));
  assertThat(dataFileReader.hasNext(), is(false));
  dataFileReader.close();
}
Example 2
Source File: AvroKeyValueSinkWriter.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private CodecFactory getCompressionCodec(Map<String, String> conf) {
	if (getBoolean(conf, CONF_COMPRESS, false)) {
		int deflateLevel = getInt(conf, CONF_DEFLATE_LEVEL, CodecFactory.DEFAULT_DEFLATE_LEVEL);
		int xzLevel = getInt(conf, CONF_XZ_LEVEL, CodecFactory.DEFAULT_XZ_LEVEL);

		String outputCodec = conf.get(CONF_COMPRESS_CODEC);

		if (DataFileConstants.DEFLATE_CODEC.equals(outputCodec)) {
			return CodecFactory.deflateCodec(deflateLevel);
		} else if (DataFileConstants.XZ_CODEC.equals(outputCodec)) {
			return CodecFactory.xzCodec(xzLevel);
		} else {
			return CodecFactory.fromString(outputCodec);
		}
	}
	return CodecFactory.nullCodec();
}
Example 3
Source File: AvroKeyValueSinkWriter.java    From flink with Apache License 2.0 6 votes vote down vote up 
private CodecFactory getCompressionCodec(Map<String, String> conf) {
	if (getBoolean(conf, CONF_COMPRESS, false)) {
		int deflateLevel = getInt(conf, CONF_DEFLATE_LEVEL, CodecFactory.DEFAULT_DEFLATE_LEVEL);
		int xzLevel = getInt(conf, CONF_XZ_LEVEL, CodecFactory.DEFAULT_XZ_LEVEL);

		String outputCodec = conf.get(CONF_COMPRESS_CODEC);

		if (DataFileConstants.DEFLATE_CODEC.equals(outputCodec)) {
			return CodecFactory.deflateCodec(deflateLevel);
		} else if (DataFileConstants.XZ_CODEC.equals(outputCodec)) {
			return CodecFactory.xzCodec(xzLevel);
		} else {
			return CodecFactory.fromString(outputCodec);
		}
	}
	return CodecFactory.nullCodec();
}
Example 4
Source File: AbstractKiteConvertProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
protected CodecFactory getCodecFactory(String property) {
    CodecType type = CodecType.valueOf(property);
    switch (type) {
    case BZIP2:
        return CodecFactory.bzip2Codec();
    case DEFLATE:
        return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL);
    case NONE:
        return CodecFactory.nullCodec();
    case LZO:
        return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL);
    case SNAPPY:
    default:
        return CodecFactory.snappyCodec();
    }
}
Example 5
Source File: AvroKeyValueSinkWriter.java    From flink with Apache License 2.0 6 votes vote down vote up 
private CodecFactory getCompressionCodec(Map<String, String> conf) {
	if (getBoolean(conf, CONF_COMPRESS, false)) {
		int deflateLevel = getInt(conf, CONF_DEFLATE_LEVEL, CodecFactory.DEFAULT_DEFLATE_LEVEL);
		int xzLevel = getInt(conf, CONF_XZ_LEVEL, CodecFactory.DEFAULT_XZ_LEVEL);

		String outputCodec = conf.get(CONF_COMPRESS_CODEC);

		if (DataFileConstants.DEFLATE_CODEC.equals(outputCodec)) {
			return CodecFactory.deflateCodec(deflateLevel);
		} else if (DataFileConstants.XZ_CODEC.equals(outputCodec)) {
			return CodecFactory.xzCodec(xzLevel);
		} else {
			return CodecFactory.fromString(outputCodec);
		}
	}
	return CodecFactory.nullCodec();
}
Example 6
Source File: Codecs.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
public static CodecFactory avroCodec(String codec) {
  CompressionCodecName parquetCodec = parquetCodec(codec);
  switch (parquetCodec) {
    case UNCOMPRESSED:
      return CodecFactory.nullCodec();
    case SNAPPY:
      return CodecFactory.snappyCodec();
    case GZIP:
      return CodecFactory.deflateCodec(9);
    case ZSTD:
      return CodecFactory.zstandardCodec(CodecFactory.DEFAULT_ZSTANDARD_LEVEL);
    default:
      throw new IllegalArgumentException(
          "Codec incompatible with Avro: " + codec);
  }
}
Example 7
Source File: AbstractKiteConvertProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up 
protected CodecFactory getCodecFactory(String property) {
    CodecType type = CodecType.valueOf(property);
    switch (type) {
    case BZIP2:
        return CodecFactory.bzip2Codec();
    case DEFLATE:
        return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL);
    case NONE:
        return CodecFactory.nullCodec();
    case LZO:
        return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL);
    case SNAPPY:
    default:
        return CodecFactory.snappyCodec();
    }
}
Example 8
Source File: AvroRecordSetWriter.java    From nifi with Apache License 2.0 6 votes vote down vote up 
private CodecFactory getCodecFactory(String property) {
    CodecType type = CodecType.valueOf(property);
    switch (type) {
    case BZIP2:
        return CodecFactory.bzip2Codec();
    case DEFLATE:
        return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL);
    case LZO:
        return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL);
    case SNAPPY:
        return CodecFactory.snappyCodec();
    case NONE:
    default:
        return CodecFactory.nullCodec();
    }
}
Example 9
Source File: AvroUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up 
public static CodecFactory getCodecFactory(String property) {
    CodecType type = CodecType.valueOf(property);
    switch (type) {
        case BZIP2:
            return CodecFactory.bzip2Codec();
        case DEFLATE:
            return CodecFactory.deflateCodec(CodecFactory.DEFAULT_DEFLATE_LEVEL);
        case LZO:
            return CodecFactory.xzCodec(CodecFactory.DEFAULT_XZ_LEVEL);
        case SNAPPY:
            return CodecFactory.snappyCodec();
        case NONE:
        default:
            return CodecFactory.nullCodec();
    }
}
Example 10
Source File: PentahoAvroOutputFormat.java    From pentaho-hadoop-shims with Apache License 2.0 5 votes vote down vote up 
@Override
public void setCompression( COMPRESSION compression ) {
  switch ( compression ) {
    case SNAPPY:
      codecFactory = CodecFactory.snappyCodec();
      break;
    case DEFLATE:
      codecFactory = CodecFactory.deflateCodec( Deflater.DEFAULT_COMPRESSION );
      break;
    default:
      codecFactory = CodecFactory.nullCodec();
      break;
  }
}
Example 11
Source File: TestWriteAvroResultWithSchema.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@Override
protected RecordSetWriter createWriter(final Schema schema, final OutputStream out) throws IOException {
    return new WriteAvroResultWithSchema(schema, out, CodecFactory.nullCodec());
}