Java Code Examples for org.apache.hadoop.io.compress.CompressionCodec#createDecompressor()

The following examples show how to use org.apache.hadoop.io.compress.CompressionCodec#createDecompressor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CodecPool.java    From tajo with Apache License 2.0 5 votes vote down vote up 
/**
 * Get a {@link Decompressor} for the given {@link CompressionCodec} from the
 * pool or a new one.
 *
 * @param codec
 *          the <code>CompressionCodec</code> for which to get the
 *          <code>Decompressor</code>
 * @return <code>Decompressor</code> for the given
 *         <code>CompressionCodec</code> the pool or a new one
 */
public static Decompressor getDecompressor(CompressionCodec codec) {
  Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec
      .getDecompressorType());
  if (decompressor == null) {
    decompressor = codec.createDecompressor();
    LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]");
  } else {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got recycled decompressor");
    }
  }
  return decompressor;
}
Example 2
Source File: CodecPool.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up 
/**
 * Get a {@link Decompressor} for the given {@link CompressionCodec} from the
 * pool or a new one.
 *
 * @param codec
 *          the <code>CompressionCodec</code> for which to get the
 *          <code>Decompressor</code>
 * @return <code>Decompressor</code> for the given
 *         <code>CompressionCodec</code> the pool or a new one
 */
public static Decompressor getDecompressor(CompressionCodec codec) {
  Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec
      .getDecompressorType());
  if (decompressor == null) {
    decompressor = codec.createDecompressor();
    LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]");
  } else {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got recycled decompressor");
    }
  }
  return decompressor;
}
Example 3
Source File: DirectCodecFactory.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
private CodecPool(final CompressionCodec codec){
  try {
    boolean supportDirectDecompressor = codec.getClass() == DIRECT_DECOMPRESSION_CODEC_CLASS;
    compressorPool = new GenericObjectPool(new BasePoolableObjectFactory() {
      public Object makeObject() throws Exception {
        return codec.createCompressor();
      }
    }, Integer.MAX_VALUE);

    Object com = compressorPool.borrowObject();
    if (com != null) {
      cPools.put(com.getClass(), compressorPool);
      compressorPool.returnObject(com);
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
      }
    }

    decompressorPool = new GenericObjectPool(new BasePoolableObjectFactory() {
      public Object makeObject() throws Exception {
        return codec.createDecompressor();
      }
    }, Integer.MAX_VALUE);

    Object decom = decompressorPool.borrowObject();
    if (decom != null) {
      dePools.put(decom.getClass(), decompressorPool);
      decompressorPool.returnObject(decom);
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "decompressor", codec.getClass().getName()));
      }
    }

    if (supportDirectDecompressor) {
      directDecompressorPool = new GenericObjectPool(
          new BasePoolableObjectFactory() {
            public Object makeObject() throws Exception {
              return CREATE_DIRECT_DECOMPRESSOR_METHOD.invoke(DIRECT_DECOMPRESSION_CODEC_CLASS);
            }
          }, Integer.MAX_VALUE);

      Object ddecom = directDecompressorPool.borrowObject();
      if (ddecom != null) {
        directDePools.put(ddecom.getClass(), directDecompressorPool);
        directDecompressorPool.returnObject(ddecom);

      } else {
        supportDirectDecompressor = false;
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
        }
      }

    } else {
      directDecompressorPool = null;
    }

    this.supportDirectDecompressor = supportDirectDecompressor;
  } catch (Exception e) {
    throw new ParquetCompressionCodecException("Error creating compression codec pool.", e);
  }
}
Example 4
Source File: DataSegmentReader.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 * May throw EOFException if InputStream does not have a
 * complete data segment.
 *
 * NOTE: This class holds reference to the Decompressor in
 * the decompressorCache, until the return value of
 * getInputStream() is closed.
 *
 * @param decompressorCache
 * @throws EmptyDataSegmentException  if there is nothing to read.
 * @throws EOFException  if the data segment is not complete.
 */
DataSegmentReader(DataInputStream in, Configuration conf,
    HashMap<Text, Decompressor> decompressorCache)
    throws EmptyDataSegmentException, EOFException,
    ClassNotFoundException, IOException {

  // Read from DataInputStream
  // 1. Read length
  int length = 0;
  try {
    length = in.readInt();
  } catch (EOFException e) {
    throw new EmptyDataSegmentException();
  }

  // 2. Read codec
  int codecNameUTF8Length = in.readShort();
  byte[] codecNameUTF8 = new byte[codecNameUTF8Length];
  in.readFully(codecNameUTF8);
  Text codecNameText = new Text(codecNameUTF8);
  // 3. read CRC32 (only present when uncompressed)
  boolean hasCrc32 = (codecNameUTF8Length == 0);
  long crc32Value = 0;
  if (hasCrc32) {
    crc32Value = in.readLong();
  }
  // 4. read data
  byte[] storedData
      = new byte[length - (hasCrc32 ? 8 : 0)/*crc32*/
                 - 2/*codec length*/ - codecNameUTF8Length];
  in.readFully(storedData);

  // Verify the checksum
  if (hasCrc32) {
    CRC32 crc32 = new CRC32();
    crc32.update(storedData);
    if (crc32.getValue() != crc32Value) {
      throw new CorruptedDataException("Corrupted data segment with length " + length
          + " crc32 expected " + crc32Value + " but got " + crc32.getValue());
    }
  }

  // Uncompress the data if needed
  if (codecNameUTF8Length == 0) {
    // no compression
    uncompressedData = new ByteArrayInputStream(storedData);
  } else {
    CompressionCodec codec = getCodecFromName(codecNameText, conf);
    Decompressor decompressor = null;
    if (decompressorCache != null) {
      // Create decompressor and add to cache if needed.
      decompressor = decompressorCache.get(codecNameText);
      if (decompressor == null) {
        decompressor = codec.createDecompressor();
      } else {
        decompressor.reset();
      }
    }
    if (decompressor == null) {
      uncompressedData = codec.createInputStream(new ByteArrayInputStream(storedData));
    } else {
      uncompressedData = codec.createInputStream(new ByteArrayInputStream(storedData),
          decompressor);
    }
  }
}