Java Code Examples for org.apache.hadoop.fs.FileSystem#closeAllForUGI()

The following examples show how to use org.apache.hadoop.fs.FileSystem#closeAllForUGI() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ContainerLocalizer.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected void closeFileSystems(UserGroupInformation ugi) {
  try {
    FileSystem.closeAllForUGI(ugi);
  } catch (IOException e) {
    LOG.warn("Failed to close filesystems: ", e);
  }
}
Example 2
Source File: LogAggregationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected void closeFileSystems(final UserGroupInformation userUgi) {
  try {
    FileSystem.closeAllForUGI(userUgi);
  } catch (IOException e) {
    LOG.warn("Failed to close filesystems: ", e);
  }
}
Example 3
Source File: TestFileSystemCaching.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCloseAllForUGI() throws Exception {
  final Configuration conf = new Configuration();
  conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
  UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo");
  FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  //Now we should get the cached filesystem
  FileSystem fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  assertSame(fsA, fsA1);
  
  FileSystem.closeAllForUGI(ugiA);
  
  //Now we should get a different (newly created) filesystem
  fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  assertNotSame(fsA, fsA1);
}
Example 4
Source File: ContainerLocalizer.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected void closeFileSystems(UserGroupInformation ugi) {
  try {
    FileSystem.closeAllForUGI(ugi);
  } catch (IOException e) {
    LOG.warn("Failed to close filesystems: ", e);
  }
}
Example 5
Source File: LogAggregationService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected void closeFileSystems(final UserGroupInformation userUgi) {
  try {
    FileSystem.closeAllForUGI(userUgi);
  } catch (IOException e) {
    LOG.warn("Failed to close filesystems: ", e);
  }
}
Example 6
Source File: TestFileSystemCaching.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCloseAllForUGI() throws Exception {
  final Configuration conf = new Configuration();
  conf.set("fs.cachedfile.impl", FileSystem.getFileSystemClass("file", null).getName());
  UserGroupInformation ugiA = UserGroupInformation.createRemoteUser("foo");
  FileSystem fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  //Now we should get the cached filesystem
  FileSystem fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  assertSame(fsA, fsA1);
  
  FileSystem.closeAllForUGI(ugiA);
  
  //Now we should get a different (newly created) filesystem
  fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
    @Override
    public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"), conf);
    }
  });
  assertNotSame(fsA, fsA1);
}
Example 7
Source File: TestHStore.java    From hbase with Apache License 2.0 4 votes vote down vote up 
@Test
public void testHandleErrorsInFlush() throws Exception {
  LOG.info("Setting up a faulty file system that cannot write");

  final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  User user = User.createUserForTesting(conf,
      "testhandleerrorsinflush", new String[]{"foo"});
  // Inject our faulty LocalFileSystem
  conf.setClass("fs.file.impl", FaultyFileSystem.class,
      FileSystem.class);
  user.runAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws Exception {
      // Make sure it worked (above is sensitive to caching details in hadoop core)
      FileSystem fs = FileSystem.get(conf);
      assertEquals(FaultyFileSystem.class, fs.getClass());

      // Initialize region
      init(name.getMethodName(), conf);

      LOG.info("Adding some data");
      store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null);
      store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null);
      store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null);

      LOG.info("Before flush, we should have no files");

      Collection<StoreFileInfo> files =
        store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
      assertEquals(0, files != null ? files.size() : 0);

      //flush
      try {
        LOG.info("Flushing");
        flush(1);
        fail("Didn't bubble up IOE!");
      } catch (IOException ioe) {
        assertTrue(ioe.getMessage().contains("Fault injected"));
      }

      LOG.info("After failed flush, we should still have no files!");
      files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
      assertEquals(0, files != null ? files.size() : 0);
      store.getHRegion().getWAL().close();
      return null;
    }
  });
  FileSystem.closeAllForUGI(user.getUGI());
}
Example 8
Source File: ContainerRunnerImpl.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Override
public ContainerExecutionResult call() throws Exception {

  // TODO Consolidate this code with TezChild.
  StopWatch sw = new StopWatch().start();
  UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser());
  taskUgi.addCredentials(credentials);

  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
  String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
      TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
  serviceConsumerMetadata.put(auxiliaryService,
      TezCommonUtils.convertJobTokenToBytes(jobToken));
  Multimap<String, String> startedInputsMap = HashMultimap.create();

  UserGroupInformation taskOwner =
      UserGroupInformation.createRemoteUser(request.getTokenIdentifier());
  final InetSocketAddress address =
      NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
  SecurityUtil.setTokenService(jobToken, address);
  taskOwner.addToken(jobToken);
  umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
    @Override
    public TezTaskUmbilicalProtocol run() throws Exception {
      return RPC.getProxy(TezTaskUmbilicalProtocol.class,
          TezTaskUmbilicalProtocol.versionID, address, conf);
    }
  });
  // TODO Stop reading this on each request.
  taskReporter = new TaskReporter(
      umbilical,
      conf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS,
          TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT),
      conf.getLong(
          TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS,
          TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT),
      conf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT,
          TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT),
      new AtomicLong(0),
      request.getContainerIdString());

  TezCommonUtils.logCredentials(LOG, taskUgi.getCredentials(), "taskUgi");
  taskRunner = new TezTaskRunner2(conf, taskUgi, localDirs,
      ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()),
      request.getAppAttemptNumber(),
      serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
      pid,
      executionContext, memoryAvailable, false, new DefaultHadoopShim(), sharedExecutor);

  boolean shouldDie;
  try {
    TaskRunner2Result result = taskRunner.run();
    LOG.info("TaskRunner2Result: {}", result);
    shouldDie = result.isContainerShutdownRequested();
    if (shouldDie) {
      LOG.info("Got a shouldDie notification via heartbeats. Shutting down");
      return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
          "Asked to die by the AM");
    }
    if (result.getError() != null) {
      Throwable e = result.getError();
      return new ContainerExecutionResult(
          ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
          e, "TaskExecutionFailure: " + e.getMessage());
    }
  } finally {
    FileSystem.closeAllForUGI(taskUgi);
  }
  LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
      sw.stop().now(TimeUnit.MILLISECONDS));
  return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
      null);
}
Example 9
Source File: UGIProvider.java    From pxf with Apache License 2.0 2 votes vote down vote up 
/**
 * Wrapper for {@link FileSystem}.closeAllForUGI method.
 *
 * @param ugi the {@link UserGroupInformation} whose filesystem resources we want to free.
 * @throws IOException
 */
void destroy(UserGroupInformation ugi) throws IOException {
    FileSystem.closeAllForUGI(ugi);
}