org.apache.hadoop.util.NativeCodeLoader Java Examples

/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
 

@Test(timeout=20000)
public void testBZip2NativeCodec() throws IOException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
      conf.set("io.compression.codec.bzip2.library", "java-builtin");
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

@Test
public void testLz4Codec() throws IOException {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Lz4Codec.isNativeCodeLoaded()) {
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          false);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          true);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
    } else {
      Assert.fail("Native hadoop library available but lz4 not");
    }
  }
}
 

@Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException, 
                      ClassNotFoundException, InstantiationException, 
                      IllegalAccessException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      sequenceFileCodecTest(conf, 0, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 100, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 200000, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 
                            1000000);
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

@Test
public void testTestCompression() {
  assertTrue(CompressionTest.testCompression("NONE"));
  assertTrue(CompressionTest.testCompression("GZ"));

  if (NativeCodeLoader.isNativeCodeLoaded()) {
    nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
    nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
    nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
    nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
    nativeCodecTest("ZSTD", "zstd", "org.apache.hadoop.io.compress.ZStandardCodec");
  } else {
    // Hadoop nativelib is not available
    LOG.debug("Native code not loaded");
    assertFalse(CompressionTest.testCompression("LZO"));
    assertFalse(CompressionTest.testCompression("LZ4"));
    assertFalse(CompressionTest.testCompression("SNAPPY"));
    assertFalse(CompressionTest.testCompression("BZIP2"));
    assertFalse(CompressionTest.testCompression("ZSTD"));
  }
}
 

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compress Compress data?
 * @param blockCompress Compress blocks?
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, boolean compress, boolean blockCompress,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else {
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param conf The configuration.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compressionType The compression type.
 * @param codec The compression codec.
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, CompressionType compressionType,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if ((codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (compressionType == CompressionType.NONE) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compressionType == CompressionType.RECORD) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else if (compressionType == CompressionType.BLOCK){
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 

/**
 * Tests that internal renames are done using native code on platforms that
 * have it.  The native rename includes more detailed information about the
 * failure, which can be useful for troubleshooting.
 */
@Test
public void testDoPreUpgradeIOError() throws IOException {
  File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
  List<URI> editUris = Collections.singletonList(storageDir.toURI());
  NNStorage storage = setupEdits(editUris, 5);
  StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
  assertNotNull(sd);
  // Change storage directory so that renaming current to previous.tmp fails.
  FileUtil.setWritable(storageDir, false);
  FileJournalManager jm = null;
  try {
    jm = new FileJournalManager(conf, sd, storage);
    exception.expect(IOException.class);
    if (NativeCodeLoader.isNativeCodeLoaded()) {
      exception.expectMessage("failure in native rename");
    }
    jm.doPreUpgrade();
  } finally {
    IOUtils.cleanup(LOG, jm);
    // Restore permissions on storage directory and make sure we can delete.
    FileUtil.setWritable(storageDir, true);
    FileUtil.fullyDelete(storageDir);
  }
}
 

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compress Compress data?
 * @param blockCompress Compress blocks?
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, boolean compress, boolean blockCompress,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else {
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 

/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 

/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 

@Test
public void testNativeCodeLoaded() {
  if (requireTestJni() == false) {
    LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
    return;
  }
  if (!NativeCodeLoader.isNativeCodeLoaded()) {
    fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
        "libhadoop.so was not loaded.");
  }
  assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
  // library names are depended on platform and build envs
  // so just check names are available
  assertFalse(ZlibFactory.getLibraryName().isEmpty());
  if (NativeCodeLoader.buildSupportsSnappy()) {
    assertFalse(SnappyCodec.getLibraryName().isEmpty());
  }
  if (NativeCodeLoader.buildSupportsOpenssl()) {
    assertFalse(OpensslCipher.getLibraryName().isEmpty());
  }
  assertFalse(Lz4Codec.getLibraryName().isEmpty());
  LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
 

@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
}
 

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * @param conf The configuration.
 * @param out The stream on top which the writer is to be constructed.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param compressionType The compression type.
 * @param codec The compression codec.
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer
  createWriter(Configuration conf, FSDataOutputStream out, 
               Class keyClass, Class valClass, CompressionType compressionType,
               CompressionCodec codec, Metadata metadata)
  throws IOException {
  if ((codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (compressionType == CompressionType.NONE) {
    writer = new Writer(conf, out, keyClass, valClass, metadata);
  } else if (compressionType == CompressionType.RECORD) {
    writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  } else if (compressionType == CompressionType.BLOCK){
    writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
  }
  
  return writer;
}
 

@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
  cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
}
 

private static boolean isNativeSnappyLoadable() {
  boolean snappyAvailable = false;
  boolean loaded = false;
  try {
    System.loadLibrary("snappy");
    logger.warn("Snappy native library is available");
    snappyAvailable = true;
    boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded();
    loaded = snappyAvailable && hadoopNativeAvailable;
    if (loaded) {
      logger.info("Snappy native library loaded");
    } else {
      logger.warn("Snappy native library not loaded");
    }
  } catch (Throwable t) {
    logger.warn("Failed to load snappy: ", t);
    return false;
  }
  return loaded;
}
 

/**
 * Method for compressor availability check
 */
private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) {
  Compressor compressor = pair.compressor;

  if (compressor.getClass().isAssignableFrom(Lz4Compressor.class)
          && (NativeCodeLoader.isNativeCodeLoaded()))
    return true;

  else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
          && NativeCodeLoader.isNativeCodeLoaded())
    return true;

  else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
    return ZlibFactory.isNativeZlibLoaded(new Configuration());
  }              
  else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)
          && isNativeSnappyLoadable())
    return true;
  
  return false;      
}
 

@Test(timeout=20000)
public void testBZip2NativeCodec() throws IOException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
      conf.set("io.compression.codec.bzip2.library", "java-builtin");
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

@Test
public void testLz4Codec() throws IOException {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Lz4Codec.isNativeCodeLoaded()) {
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          false);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
      conf.setBoolean(
          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
          true);
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.Lz4Codec");
    } else {
      Assert.fail("Native hadoop library available but lz4 not");
    }
  }
}
 

@Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException, 
                      ClassNotFoundException, InstantiationException, 
                      IllegalAccessException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      sequenceFileCodecTest(conf, 0, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 100, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 200000, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 
                            1000000);
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
}
 

private static boolean isNativeSnappyLoadable() {
  boolean snappyAvailable = false;
  boolean loaded = false;
  try {
    System.loadLibrary("snappy");
    logger.warn("Snappy native library is available");
    snappyAvailable = true;
    boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded();
    loaded = snappyAvailable && hadoopNativeAvailable;
    if (loaded) {
      logger.info("Snappy native library loaded");
    } else {
      logger.warn("Snappy native library not loaded");
    }
  } catch (Throwable t) {
    logger.warn("Failed to load snappy: ", t);
    return false;
  }
  return loaded;
}
 

@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
  if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
    LOG.warn("Skipping since test was not run with -Pnative flag");
    Assume.assumeTrue(false);
  }
  if (!NativeCodeLoader.buildSupportsOpenssl()) {
    LOG.warn("Skipping test since openSSL library not loaded");
    Assume.assumeTrue(false);
  }
  Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
  cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
  // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
  for(int i = 0; i < 8; i++) {
    iv[8 + i] = (byte) 0xff;
  }
  cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
  cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
}
 

/**
 * Are the native snappy libraries loaded & initialized?
 */
public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsSnappy()) {
      throw new RuntimeException("native snappy library not available: " +
          "this version of libhadoop was built without " +
          "snappy support.");
    }
    if (!SnappyCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyCompressor has not been loaded.");
    }
    if (!SnappyDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native snappy library not available: " +
          "SnappyDecompressor has not been loaded.");
    }
}
 

/**
   * Construct the preferred type of 'raw' SequenceFile Writer.
   * @param fs The configured filesystem. 
   * @param conf The configuration.
   * @param file The name of the file. 
   * @param keyClass The 'key' type.
   * @param valClass The 'value' type.
   * @param compress Compress data?
   * @param blockCompress Compress blocks?
   * @param codec The compression codec.
   * @param progress
   * @param metadata The metadata of the file.
   * @return Returns the handle to the constructed SequenceFile Writer.
   * @throws IOException
   */
  private static Writer
  createWriter(FileSystem fs, Configuration conf, Path file, 
               Class keyClass, Class valClass, 
               boolean compress, boolean blockCompress,
               CompressionCodec codec, Progressable progress, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, 
                                      codec, progress, metadata);
  } else {
    writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, 
                                     codec, progress, metadata);
  }
  
  return writer;
}
 

/**
 * Check if native-bzip2 code is loaded & initialized correctly and 
 * can be loaded for this job.
 * 
 * @param conf configuration
 * @return <code>true</code> if native-bzip2 is loaded & initialized 
 *         and can be loaded for this job, else <code>false</code>
 */
public static boolean isNativeBzip2Loaded(Configuration conf) {
  String libname = conf.get("io.compression.codec.bzip2.library", 
                            "system-native");
  if (!bzip2LibraryName.equals(libname)) {
    nativeBzip2Loaded = false;
    bzip2LibraryName = libname;
    if (libname.equals("java-builtin")) {
      LOG.info("Using pure-Java version of bzip2 library");
    } else if (conf.getBoolean(
              CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, 
              CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
        NativeCodeLoader.isNativeCodeLoaded()) {
      try {
        // Initialize the native library.
        Bzip2Compressor.initSymbols(libname);
        Bzip2Decompressor.initSymbols(libname);
        nativeBzip2Loaded = true;
        LOG.info("Successfully loaded & initialized native-bzip2 library " +
                 libname);
      } catch (Throwable t) {
        LOG.warn("Failed to load/initialize native-bzip2 library " + 
                 libname + ", will use pure-Java version");
      }
    }
  }
  return nativeBzip2Loaded;
}
 

public JniBasedUnixGroupsNetgroupMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsNetgroupMapping();
  } else {
    LOG.info("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsNetgroupMapping();
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
 

public JniBasedUnixGroupsMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsMapping();
  } else {
    PerformanceAdvisory.LOG.debug("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsMapping();
  }
  if (LOG.isDebugEnabled()){
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
 

/**
   * Construct the preferred type of 'raw' SequenceFile Writer.
   * @param fs The configured filesystem. 
   * @param conf The configuration.
   * @param file The name of the file. 
   * @param keyClass The 'key' type.
   * @param valClass The 'value' type.
   * @param compress Compress data?
   * @param blockCompress Compress blocks?
   * @param codec The compression codec.
   * @param progress
   * @param metadata The metadata of the file.
   * @return Returns the handle to the constructed SequenceFile Writer.
   * @throws IOException
   */
  private static Writer
  createWriter(FileSystem fs, Configuration conf, Path file, 
               Class keyClass, Class valClass, 
               boolean compress, boolean blockCompress,
               CompressionCodec codec, Progressable progress, Metadata metadata)
  throws IOException {
  if (codec != null && (codec instanceof GzipCodec) && 
      !NativeCodeLoader.isNativeCodeLoaded() && 
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (!compress) {
    writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
  } else if (compress && !blockCompress) {
    writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, 
                                      codec, progress, metadata);
  } else {
    writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, 
                                     codec, progress, metadata);
  }
  
  return writer;
}
 

/**
 * Construct the preferred type of SequenceFile Writer.
 * @param fs The configured filesystem.
 * @param conf The configuration.
 * @param name The name of the file.
 * @param keyClass The 'key' type.
 * @param valClass The 'value' type.
 * @param bufferSize buffer size for the underlaying outputstream.
 * @param replication replication factor for the file.
 * @param blockSize block size for the file.
 * @param compressionType The compression type.
 * @param codec The compression codec.
 * @param progress The Progressable object to track progress.
 * @param metadata The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer
  createWriter(FileSystem fs, Configuration conf, Path name,
               Class keyClass, Class valClass, int bufferSize,
               short replication, long blockSize,
               CompressionType compressionType, CompressionCodec codec,
               Progressable progress, Metadata metadata) throws IOException {
  if ((codec instanceof GzipCodec) &&
      !NativeCodeLoader.isNativeCodeLoaded() &&
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop code!");
  }

  Writer writer = null;

  if (compressionType == CompressionType.NONE) {
    writer = new Writer(fs, conf, name, keyClass, valClass,
                        bufferSize, replication, blockSize,
                        progress, metadata);
  } else if (compressionType == CompressionType.RECORD) {
    writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass,
                                      bufferSize, replication, blockSize,
                                      codec, progress, metadata);
  } else if (compressionType == CompressionType.BLOCK){
    writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass,
                                     bufferSize, replication, blockSize,
                                     codec, progress, metadata);
  }

  return writer;
}