Java Code Examples for org.apache.hadoop.fs.FileSystem#newInstance()

@Test
public void testFsUniqueness() throws Exception {
  final Configuration conf = new Configuration();
  conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
  // multiple invocations of FileSystem.get return the same object.
  FileSystem fs1 = FileSystem.get(conf);
  FileSystem fs2 = FileSystem.get(conf);
  assertTrue(fs1 == fs2);

  // multiple invocations of FileSystem.newInstance return different objects
  fs1 = FileSystem.newInstance(new URI("cachedfile://a"), conf, "bar");
  fs2 = FileSystem.newInstance(new URI("cachedfile://a"), conf, "bar");
  assertTrue(fs1 != fs2 && !fs1.equals(fs2));
  fs1.close();
  fs2.close();
}
 

AppPackage newAppPackageInstance(URI uri, boolean suppressOutput) throws IOException
{
  PrintStream outputStream = suppressOutput ? suppressOutput() : null;
  try {
    final String scheme = uri.getScheme();
    if (scheme == null || scheme.equals("file")) {
      return new AppPackage(new FileInputStream(new File(expandFileName(uri.getPath(), true))), true);
    } else {
      try (FileSystem fs = FileSystem.newInstance(uri, conf)) {
        return new AppPackage(fs.open(new Path(uri.getPath())), true);
      }
    }
  } finally {
    if (outputStream != null) {
      restoreOutput(outputStream);
    }
  }
}
 

public static FileSystem newFileSystemInstance(Configuration conf) throws IOException
{
  String dfsRootDir = conf.get(DT_DFS_ROOT_DIR);
  if (StringUtils.isBlank(dfsRootDir)) {
    return FileSystem.newInstance(conf);
  } else {
    if (dfsRootDir.contains(DT_DFS_USER_NAME)) {
      dfsRootDir = dfsRootDir.replace(DT_DFS_USER_NAME, UserGroupInformation.getLoginUser().getShortUserName());
      conf.set(DT_DFS_ROOT_DIR, dfsRootDir);
    }
    try {
      return FileSystem.newInstance(new URI(dfsRootDir), conf);
    } catch (URISyntaxException ex) {
      LOG.warn("{} is not a valid URI. Returning the default filesystem", dfsRootDir, ex);
      return FileSystem.newInstance(conf);
    }
  }
}
 

@Test
public void testResetContextClassLoaderResetsToInitialClassLoader() throws Exception
{
  // Setup
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.newInstance(conf);
  StramAppLauncher appLauncher = new StramAppLauncher(fs, conf);

  Whitebox.setInternalState(appLauncher, "launchDependencies", new LinkedHashSet<URL>());

  // Get initial contextClassLoader
  ClassLoader initialClassLoader = Thread.currentThread().getContextClassLoader();

  appLauncher.loadDependencies();
  appLauncher.resetContextClassLoader();

  ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
  Assert.assertSame(initialClassLoader, currentClassLoader);
}
 

private static FileSystem getWriterFsUsingKeytab(State state, URI uri)
    throws IOException {
  FileSystem fs = FileSystem.newInstance(uri, new Configuration());
  try {
    Preconditions.checkArgument(state.contains(ConfigurationKeys.FS_PROXY_AS_USER_NAME),
        "Missing required property " + ConfigurationKeys.FS_PROXY_AS_USER_NAME);
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS),
        "Missing required property " + ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
        "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
    String user = state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME);
    String superUser = state.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
    Path keytabLocation = new Path(state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION));
    return ProxiedFileSystemCache.fromKeytab().userNameToProxyAs(user).fsURI(uri)
        .superUserKeytabLocation(keytabLocation).superUserName(superUser).conf(HadoopUtils.newConfiguration())
        .referenceFS(fs).build();
  } catch (ExecutionException e) {
    throw new IOException(e);
  }
}
 

@Override
public void setup(Context ctx) throws IOException, InterruptedException
{
  super.setup(ctx);

  mos = new MultipleOutputs<>(ctx);

  FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
  storage = new HadoopFileSystemStore(fs);
  String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
  Query query = storage.recall(queryDir, Query.class);
  QueryInfo queryInfo = query.getQueryInfo();

  outputFile = ctx.getConfiguration().get("pirMR.outputFile");

  response = new Response(queryInfo);
}
 

@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithProjection(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_PROJECTION, projectionSchema.toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    fsConfig.setReader(getReader(fsConfig.getFs(), fsConfig.getDataFile(), readerConfig));
    while (fsConfig.getReader().hasNext()) {
        Struct record = fsConfig.getReader().next();
        assertNotNull(record.schema().field(FIELD_INDEX));
        assertNotNull(record.schema().field(FIELD_NAME));
        assertNull(record.schema().field(FIELD_SURNAME));
    }
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    assertThrows(DataException.class, () -> readAllData(fsConfig));
}
 

@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithSchema(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_SCHEMA, readerSchema.toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    readAllData(fsConfig);
}
 

@SuppressWarnings("deprecation")
@Before
  public void setUp() throws Exception {
      Path hbaseRootDir = TEST_UTIL.getDataTestDir();
      TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());

      FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
      HRegionInfo hri = new HRegionInfo(tableName, null, null, false);
      Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
      WAL wal = walFactory.getWAL(Bytes.toBytes("logs"));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
      htd.addFamily(a);
      
      r1 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 1;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
        
      r2 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 2;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
  }
 

private boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException
{
  boolean result;
  long now = System.currentTimeMillis();
  Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath());
  try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) {
    List<String> strings = Lists.newArrayList();
    while (System.currentTimeMillis() - now < timeout) {
      if (fs.exists(outDir)) {
        File file = new File(outputFolder);
        if (file.list().length > 0) {
          File file1 = new File(outputFolder + file.list()[0]);
          strings = FileUtils.readLines(file1);
          if (strings.size() != 0) {
            break;
          }
        }
      }

      Thread.sleep(500);
    }

    result = fs.exists(outDir) && (strings.size() != 0);
  }

  return result;
}
 

@ParameterizedTest
@MethodSource("fileSystemConfigProvider")
public void readerWithInvalidSchema(ReaderFsTestConfig fsConfig) throws IOException {
    Map<String, Object> readerConfig = getReaderConfig();
    readerConfig.put(ParquetFileReader.FILE_READER_PARQUET_SCHEMA, Schema.create(Schema.Type.STRING).toString());
    readerConfig.put(AgnosticFileReader.FILE_READER_AGNOSTIC_EXTENSIONS_PARQUET, getFileExtension());
    FileSystem testFs = FileSystem.newInstance(fsConfig.getFsUri(), new Configuration());
    fsConfig.setReader(getReader(testFs, fsConfig.getDataFile(), readerConfig));
    try {
        readAllData(fsConfig);
    } catch (Exception e) {
        assertEquals(ConnectException.class, e.getClass());
        assertEquals(AvroRuntimeException.class, e.getCause().getClass());
    }
}
 

@Override
public void setup(Context ctx) throws IOException, InterruptedException
{
  super.setup(ctx);

  outputValue = new Text();
  mos = new MultipleOutputs<>(ctx);

  FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
  String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
  query = new HadoopFileSystemStore(fs).recall(queryDir, Query.class);
}
 

private void configFs(Map<String, Object> customConfigs) throws IOException {
    for (String uri : this.conf.getFsUris()) {
        Configuration fsConfig = new Configuration();
        customConfigs.entrySet().stream()
                .filter(entry -> entry.getKey().startsWith(FsSourceTaskConfig.POLICY_PREFIX_FS))
                .forEach(entry -> fsConfig.set(entry.getKey().replace(FsSourceTaskConfig.POLICY_PREFIX_FS, ""),
                        (String) entry.getValue()));

        Path workingDir = new Path(convert(uri));
        FileSystem fs = FileSystem.newInstance(workingDir.toUri(), fsConfig);
        fs.setWorkingDirectory(workingDir);
        this.fileSystems.add(fs);
    }
}
 

private FileLoader(String file, String backupResource) throws IOException
{
  Configuration conf = new Configuration();
  this.backupResource = backupResource;
  this.filePath = new Path(file);

  try {
    this.fs = FileSystem.newInstance(filePath.toUri(), conf);
  } catch (IOException ex) {
    LOG.error("Exception {}", ex);
  }
}
 

private static String getHeaderFromFile(Configuration hadoopConf, Path path) throws IOException {
  String header;
  try (FileSystem fs = FileSystem.newInstance(hadoopConf);
       BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path), StandardCharsets.UTF_8))) {
    // read one line - the header
    header = br.readLine();
  }
  return header;
}
 

public void copyInitialState(Path origAppDir) throws IOException
{
  // locate previous snapshot
  long copyStart = System.currentTimeMillis();
  String newAppDir = this.dag.assertAppPath();

  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
  // read snapshot against new dependencies
  Object snapshot = recoveryHandler.restore();
  if (snapshot == null) {
    throw new IllegalArgumentException("No previous application state found in " + origAppDir);
  }
  InputStream logIs = recoveryHandler.getLog();

  // modify snapshot state to switch app id
  ((StreamingContainerManager.CheckpointState)snapshot).setApplicationId(this.dag, conf);
  Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);

  FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
  // remove the path that was created by the storage agent during deserialization and replacement
  fs.delete(checkpointPath, true);

  // write snapshot to new location
  recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
  recoveryHandler.save(snapshot);
  OutputStream logOs = recoveryHandler.rotateLog();
  IOUtils.copy(logIs, logOs);
  logOs.flush();
  logOs.close();
  logIs.close();

  List<String> excludeDirs = Arrays.asList(LogicalPlan.SUBDIR_CHECKPOINTS, LogicalPlan.SUBDIR_EVENTS, LogicalPlan.SUBDIR_STATS);
  // copy sub directories that are not present in target
  FileStatus[] lFiles = fs.listStatus(origAppDir);

  // In case of MapR/MapR-FS, f.getPath().toString() returns path as maprfs:///<orig app dir>
  // whereas origAppDir.toString & newAppDir are in maprfs:/<orig or new app dir> format
  // e.g.
  // f.getPath().toString -> maprfs:///user/dtadmin/datatorrent/apps/application_1481890072066_0004/checkpoints
  // origAppDir -> maprfs:/user/dtadmin/datatorrent/apps/application_1481890072066_0004
  // newAppDir -> maprfs:/user/dtadmin/datatorrent/apps/application_1481890072066_0005

  String origAppDirPath = Path.getPathWithoutSchemeAndAuthority(origAppDir).toString();
  String newAppDirPath = Path.getPathWithoutSchemeAndAuthority(new Path(newAppDir)).toString();

  for (FileStatus f : lFiles) {
    if (f.isDirectory() && !excludeDirs.contains(f.getPath().getName())) {
      String targetPath = f.getPath().toString().replace(origAppDirPath, newAppDirPath);
      if (!fs.exists(new Path(targetPath))) {
        LOG.debug("Copying {} size {} to {}", f.getPath(), f.getLen(), targetPath);
        long start = System.currentTimeMillis();
        FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
        LOG.debug("Copying {} to {} took {} ms", f.getPath(), f.getLen(), targetPath, System.currentTimeMillis() - start);
      } else {
        LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
      }
    }
  }
  LOG.info("Copying initial state took {} ms", System.currentTimeMillis() - copyStart);
}
 

/**
 * Block Recovery when the meta file not having crcs for all chunks in block
 * file
 */
@Test
public void testBlockRecoveryWithLessMetafile() throws Exception {
  Configuration conf = new Configuration();
  conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
      UserGroupInformation.getCurrentUser().getShortUserName());
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Path file = new Path("/testRecoveryFile");
  DistributedFileSystem dfs = cluster.getFileSystem();
  FSDataOutputStream out = dfs.create(file);
  int count = 0;
  while (count < 2 * 1024 * 1024) {
    out.writeBytes("Data");
    count += 4;
  }
  out.hsync();
  // abort the original stream
  ((DFSOutputStream) out.getWrappedStream()).abort();

  LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(
      file.toString(), 0, count);
  ExtendedBlock block = locations.get(0).getBlock();
  DataNode dn = cluster.getDataNodes().get(0);
  BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null);
  File metafile = new File(localPathInfo.getMetaPath());
  assertTrue(metafile.exists());

  // reduce the block meta file size
  RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
  raf.setLength(metafile.length() - 20);
  raf.close();

  // restart DN to make replica to RWR
  DataNodeProperties dnProp = cluster.stopDataNode(0);
  cluster.restartDataNode(dnProp, true);

  // try to recover the lease
  DistributedFileSystem newdfs = (DistributedFileSystem) FileSystem
      .newInstance(cluster.getConfiguration(0));
  count = 0;
  while (++count < 10 && !newdfs.recoverLease(file)) {
    Thread.sleep(1000);
  }
  assertTrue("File should be closed", newdfs.recoverLease(file));

}
 

public HdfsStorage(Configuration conf,  String url) throws IOException {
  fs = FileSystem.newInstance(URI.create(url), conf);
  this.conf = conf;
  this.url = url;
}
 

/**
 * Get another FileSystem instance that is different from FileSystem.get(conf).
 * This simulating different threads working on different FileSystem instances.
 */
public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException {
  return FileSystem.newInstance(getURI(nnIndex), nameNodes[nnIndex].conf);
}
 

/**
 *
 * @return Destination FileSystem instance
 * @throws IOException
 */
protected FileSystem getOutputFSInstance() throws IOException
{
  return FileSystem.newInstance((new Path(filePath)).toUri(), new Configuration());
}