Java Code Examples for org.apache.hadoop.fs.FileSystem#closeAll()

The following examples show how to use org.apache.hadoop.fs.FileSystem#closeAll() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFileInputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
      "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  JobConf job = new JobConf(conf);
  TextInputFormat fileInputFormat = new TextInputFormat();
  fileInputFormat.configure(job);
  InputSplit[] splits = fileInputFormat.getSplits(job, 1);
  Assert.assertEquals("Input splits are not correct", 2, splits.length);
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
Example 2
Source File: TestTrash.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public static void trashNonDefaultFS(Configuration conf) throws IOException {
  conf.set("fs.trash.interval", "10"); // 10 minute
  // attempt non-default FileSystem trash
  {
    final FileSystem lfs = FileSystem.getLocal(conf);
    Path p = TEST_DIR;
    Path f = new Path(p, "foo/bar");
    if (lfs.exists(p)) {
      lfs.delete(p, true);
    }
    try {
      f = writeFile(lfs, f);

      FileSystem.closeAll();
      FileSystem localFs = FileSystem.get(URI.create("file:///"), conf);
      Trash lTrash = new Trash(localFs, conf);
      lTrash.moveToTrash(f.getParent());
      checkTrash(localFs, lTrash.getCurrentTrashDir(), f);
    } finally {
      if (lfs.exists(p)) {
        lfs.delete(p, true);
      }
    }
  }
}
Example 3
Source File: TestDistributedFileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testFileSystemCloseAll() throws Exception {
  Configuration conf = getTestConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
  URI address = FileSystem.getDefaultUri(conf);

  try {
    FileSystem.closeAll();

    conf = getTestConfiguration();
    FileSystem.setDefaultUri(conf, address);
    FileSystem.get(conf);
    FileSystem.get(conf);
    FileSystem.closeAll();
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
Example 4
Source File: DFSLocationsRoot.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public void disconnect() {
  Thread closeThread = new Thread() {
    /* @inheritDoc */
    @Override
    public void run() {
      try {
        System.out.printf("Closing all opened File Systems...\n");
        FileSystem.closeAll();
        System.out.printf("File Systems closed\n");

      } catch (IOException ioe) {
        ioe.printStackTrace();
      }
    }
  };

  // Wait 5 seconds for the connections to be closed
  closeThread.start();
  try {
    closeThread.join(5000);

  } catch (InterruptedException ie) {
    // Ignore
  }
}
Example 5
Source File: TestFileInputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.set(FileInputFormat.INPUT_DIR, "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  Job job = Job.getInstance(conf);
  FileInputFormat<?, ?> fileInputFormat = new TextInputFormat();
  List<InputSplit> splits = fileInputFormat.getSplits(job);
  Assert.assertEquals("Input splits are not correct", 2, splits.size());
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
Example 6
Source File: DFSLocationsRoot.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
public void disconnect() {
  Thread closeThread = new Thread() {
    /* @inheritDoc */
    @Override
    public void run() {
      try {
        System.out.printf("Closing all opened File Systems...\n");
        FileSystem.closeAll();
        System.out.printf("File Systems closed\n");

      } catch (IOException ioe) {
        ioe.printStackTrace();
      }
    }
  };

  // Wait 5 seconds for the connections to be closed
  closeThread.start();
  try {
    closeThread.join(5000);

  } catch (InterruptedException ie) {
    // Ignore
  }
}
Example 7
Source File: TestStreamXmlMultipleRecords.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
Example 8
Source File: HdfsDatasetSinkConfiguration.java    From spring-cloud-stream-app-starters with Apache License 2.0 5 votes vote down vote up 
@Override
public void stop() {
	try {
		FileSystem.closeAll();
		logger.info("Closing the Hadoop FileSystem");
	} catch (IOException e) {
		logger.error("Unable to close Hadoop FileSystem", e);
	}
	this.running = false;
}
Example 9
Source File: TestBlockFixer.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
public static DistributedFileSystem getDFS(
      Configuration conf, FileSystem dfs) throws IOException {
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl",
                 "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  FileSystem.closeAll();
  return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf);
}
Example 10
Source File: CancellingTestBase.java    From stratosphere with Apache License 2.0 5 votes vote down vote up 
@After
public void stopCluster() throws Exception {
	if (this.executor != null) {
		this.executor.stop();
		this.executor = null;
		FileSystem.closeAll();
		System.gc();
	}
}
Example 11
Source File: TestStreamXmlMultipleRecords.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
Example 12
Source File: AbstractTestBase.java    From stratosphere with Apache License 2.0 5 votes vote down vote up 
@After
public void stopCluster() throws Exception {
	try {
		if (this.executor != null) {
			this.executor.stop();
			this.executor = null;
			FileSystem.closeAll();
			System.gc();
		}
	} finally {
		deleteAllTempFiles();
	}
}
Example 13
Source File: GoogleHadoopFSIntegrationTest.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up 
@After
public void after() throws IOException {
  FileSystem.closeAll();
}
Example 14
Source File: MR4CGenericOptionsParser.java    From mr4c with Apache License 2.0 4 votes vote down vote up 
/**
  * This should be called after tests to get rid of FileSystem side effects
*/
public static void cleanup() throws IOException {
	// GenericOptionsParser will cache FileSystem objects based on the config it is updating
	FileSystem.closeAll();
}
Example 15
Source File: TestAggregatedLogDeletionService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
Example 16
Source File: SafeFileOutputCommitterTest.java    From datawave with Apache License 2.0 4 votes vote down vote up 
private void testConcurrentCommitTaskWithSubDir(int version) throws Exception {
    final Job job = Job.getInstance();
    FileOutputFormat.setOutputPath(job, outDir);
    final Configuration conf = job.getConfiguration();
    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
    conf.setInt(FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);
    
    conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
    FileSystem.closeAll();
    
    final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
    final FileOutputCommitter amCommitter = new SafeFileOutputCommitter(outDir, jContext);
    amCommitter.setupJob(jContext);
    
    final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
    taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
    taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
    
    final TextOutputFormat[] tof = new TextOutputFormat[2];
    for (int i = 0; i < tof.length; i++) {
        tof[i] = new TextOutputFormat() {
            @Override
            public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
                final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context);
                return new Path(new Path(foc.getWorkPath(), SUB_DIR), getUniqueFile(context, getOutputName(context), extension));
            }
        };
    }
    
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    try {
        for (int i = 0; i < taCtx.length; i++) {
            final int taskIdx = i;
            executor.submit((Callable<Void>) () -> {
                final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
                outputCommitter.setupTask(taCtx[taskIdx]);
                final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
                writeOutput(rw, taCtx[taskIdx]);
                outputCommitter.commitTask(taCtx[taskIdx]);
                return null;
            });
        }
    } finally {
        executor.shutdown();
        while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
            LOG.info("Awaiting thread termination!");
        }
    }
    
    amCommitter.commitJob(jContext);
    final RawLocalFileSystem lfs = new RawLocalFileSystem();
    lfs.setConf(conf);
    assertFalse("Must not end up with sub_dir/sub_dir", lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));
    
    // validate output
    validateContent(OUT_SUB_DIR);
    FileUtil.fullyDelete(new File(outDir.toString()));
}
Example 17
Source File: TestHftpFileSystem.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@After
public void resetFileSystems() throws IOException {
  FileSystem.closeAll();
}
Example 18
Source File: GoogleHadoopFileSystemIntegrationTest.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up 
@Before
public void clearFileSystemCache() throws IOException {
  FileSystem.closeAll();
}
Example 19
Source File: TestAggregatedLogDeletionService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
Example 20
Source File: HadoopFileSystemsUtils.java    From ignite with Apache License 2.0 2 votes vote down vote up 
/**
 * Clears Hadoop {@link FileSystem} cache.
 *
 * @throws IOException On error.
 */
public static void clearFileSystemCache() throws IOException {
    FileSystem.closeAll();
}