org.apache.hadoop.yarn.exceptions.YarnRuntimeException Java Examples

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  ApplicationMasterProtocol instance = new AMRMProtocolTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
          ApplicationMasterProtocol.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to create server");
  } finally {
    if (server != null) {
      server.stop();
    }
  }
}
 

protected void handleStoreEvent(NodeLabelsStoreEvent event) {
  try {
    switch (event.getType()) {
    case ADD_LABELS:
      StoreNewClusterNodeLabels storeNewClusterNodeLabelsEvent =
          (StoreNewClusterNodeLabels) event;
      store.storeNewClusterNodeLabels(storeNewClusterNodeLabelsEvent
           .getLabels());
      break;
    case REMOVE_LABELS:
      RemoveClusterNodeLabels removeClusterNodeLabelsEvent =
          (RemoveClusterNodeLabels) event;
      store.removeClusterNodeLabels(removeClusterNodeLabelsEvent.getLabels());
      break;
    case STORE_NODE_TO_LABELS:
      UpdateNodeToLabelsMappingsEvent updateNodeToLabelsMappingsEvent =
          (UpdateNodeToLabelsMappingsEvent) event;
      store.updateNodeToLabelsMappings(updateNodeToLabelsMappingsEvent
          .getNodeToLabels());
      break;
    }
  } catch (IOException e) {
    LOG.error("Failed to store label modification to storage");
    throw new YarnRuntimeException(e);
  }
}
 

public static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase) {
  switch (phase) {
  case STARTING:
    return Phase.STARTING;
  case MAP:
    return Phase.MAP;
  case SHUFFLE:
    return Phase.SHUFFLE;
  case SORT:
    return Phase.SORT;
  case REDUCE:
    return Phase.REDUCE;
  case CLEANUP:
    return Phase.CLEANUP;
  }
  throw new YarnRuntimeException("Unrecognized Phase: " + phase);
}
 

public void start(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  //TODO : use fixed port ??
  InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
  InetAddress hostNameResolved = null;
  try {
    address.getAddress();
    hostNameResolved = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    throw new YarnRuntimeException(e);
  }

  server =
      rpc.getServer(protocol, this, address,
          conf, null, 1);
  server.start();
  this.bindAddress = NetUtils.getConnectAddress(server);
   super.start();
   amRunning = true;
}
 

/**
 * Get a new <code>SharedCacheChecksum</code> object based on the configurable
 * algorithm implementation
 * (see <code>yarn.sharedcache.checksum.algo.impl</code>)
 *
 * @return <code>SharedCacheChecksum</code> object
 */
public static SharedCacheChecksum getChecksum(Configuration conf) {
  Class<? extends SharedCacheChecksum> clazz =
      conf.getClass(YarnConfiguration.SHARED_CACHE_CHECKSUM_ALGO_IMPL,
      defaultAlgorithm, SharedCacheChecksum.class);
  SharedCacheChecksum checksum = instances.get(clazz);
  if (checksum == null) {
    try {
      checksum = ReflectionUtils.newInstance(clazz, conf);
      SharedCacheChecksum old = instances.putIfAbsent(clazz, checksum);
      if (old != null) {
        checksum = old;
      }
    } catch (Exception e) {
      throw new YarnRuntimeException(e);
    }
  }

  return checksum;
}
 

@Test (timeout = 30000)
public void testConfValidation() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
  try {
    scheduler.serviceInit(conf);
    fail("Exception is expected because the min memory allocation is" +
      " larger than the max memory allocation.");
  } catch (YarnRuntimeException e) {
    // Exception is expected.
    assertTrue("The thrown exception is not the expected one.",
      e.getMessage().startsWith(
        "Invalid resource scheduler memory"));
  }
}
 

protected void rebootNodeStatusUpdaterAndRegisterWithRM() {
  // Interrupt the updater.
  this.isStopped = true;

  try {
    statusUpdater.join();
    registerWithRM();
    statusUpdater = new Thread(statusUpdaterRunnable, "Node Status Updater");
    this.isStopped = false;
    statusUpdater.start();
    LOG.info("NodeStatusUpdater thread is reRegistered and restarted");
  } catch (Exception e) {
    String errorMessage = "Unexpected error rebooting NodeStatusUpdater";
    LOG.error(errorMessage, e);
    throw new YarnRuntimeException(e);
  }
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  super.serviceInit(conf);
  commitThreadCancelTimeoutMs = conf.getInt(
      MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS);
  commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS);
  try {
    fs = FileSystem.get(conf);
    JobID id = TypeConverter.fromYarn(context.getApplicationID());
    JobId jobId = TypeConverter.toYarn(id);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
    endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
 

/**
 * Function that actually creates the ApplicationId by calling the
 * ClientRMService
 * 
 * @return returns structure containing the app-id and maximum resource
 *         capabilities
 */
private NewApplication createNewApplication() {
  GetNewApplicationRequest req =
      recordFactory.newRecordInstance(GetNewApplicationRequest.class);
  GetNewApplicationResponse resp;
  try {
    resp = rm.getClientRMService().getNewApplication(req);
  } catch (YarnException e) {
    String msg = "Unable to create new app from RM web service";
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
  NewApplication appId =
      new NewApplication(resp.getApplicationId().toString(),
        new ResourceInfo(resp.getMaximumResourceCapability()));
  return appId;
}
 

protected synchronized void serviceStart() throws Exception {
  try {
    new Thread() {
      public void run() {
        nodeManagers[index].start();
      }
    }.start();
    int waitCount = 0;
    while (nodeManagers[index].getServiceState() == STATE.INITED
        && waitCount++ < 60) {
      LOG.info("Waiting for NM " + index + " to start...");
      Thread.sleep(1000);
    }
    if (nodeManagers[index].getServiceState() != STATE.STARTED) {
      // RM could have failed.
      throw new IOException("NodeManager " + index + " failed to start");
    }
    super.serviceStart();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
}
 

@Override
protected HAServiceTarget resolveTarget(String rmId) {
  Collection<String> rmIds = HAUtil.getRMHAIds(getConf());
  if (!rmIds.contains(rmId)) {
    StringBuilder msg = new StringBuilder();
    msg.append(rmId + " is not a valid serviceId. It should be one of ");
    for (String id : rmIds) {
      msg.append(id + " ");
    }
    throw new IllegalArgumentException(msg.toString());
  }
  try {
    YarnConfiguration conf = new YarnConfiguration(getConf());
    conf.set(YarnConfiguration.RM_HA_ID, rmId);
    return new RMHAServiceTarget(conf);
  } catch (IllegalArgumentException iae) {
    throw new YarnRuntimeException("Could not connect to " + rmId +
        "; the configuration for it might be missing");
  } catch (IOException ioe) {
    throw new YarnRuntimeException(
        "Could not connect to RM HA Admin for node " + rmId);
  }
}
 

@SuppressWarnings("unchecked")
private void initApp(final ApplicationId appId, String user,
    Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy,
    Map<ApplicationAccessType, String> appAcls,
    LogAggregationContext logAggregationContext) {
  ApplicationEvent eventResponse;
  try {
    verifyAndCreateRemoteLogDir(getConfig());
    initAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls,
        logAggregationContext);
    eventResponse = new ApplicationEvent(appId,
        ApplicationEventType.APPLICATION_LOG_HANDLING_INITED);
  } catch (YarnRuntimeException e) {
    LOG.warn("Application failed to init aggregation", e);
    eventResponse = new ApplicationEvent(appId,
        ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED);
  }
  this.dispatcher.getEventHandler().handle(eventResponse);
}
 

@Override
protected void serviceStart() throws Exception {
  String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(),
                        YarnConfiguration.NM_BIND_HOST,
                        WebAppUtils.getNMWebAppURLWithoutScheme(getConfig()));
  
  LOG.info("Instantiating NMWebApp at " + bindAddress);
  try {
    this.webApp =
        WebApps
          .$for("node", Context.class, this.nmContext, "ws")
          .at(bindAddress)
          .with(getConfig())
          .withHttpSpnegoPrincipalKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
          .withHttpSpnegoKeytabKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
          .start(this.nmWebApp);
    this.port = this.webApp.httpServer().getConnectorAddress(0).getPort();
  } catch (Exception e) {
    String msg = "NMWebapps failed to start.";
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
  super.serviceStart();
}
 

private synchronized void loadRMAppState(RMState rmState) throws Exception {
  List<String> childNodes = getChildrenWithRetries(rmAppRoot, false);
  for (String childNodeName : childNodes) {
    String childNodePath = getNodePath(rmAppRoot, childNodeName);
    byte[] childData = getDataWithRetries(childNodePath, false);
    if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
      // application
      if (LOG.isDebugEnabled()) {
        LOG.debug("Loading application from znode: " + childNodeName);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
      ApplicationStateDataPBImpl appState =
          new ApplicationStateDataPBImpl(
              ApplicationStateDataProto.parseFrom(childData));
      if (!appId.equals(
          appState.getApplicationSubmissionContext().getApplicationId())) {
        throw new YarnRuntimeException("The child node name is different " +
            "from the application id");
      }
      rmState.appState.put(appId, appState);
      loadApplicationAttemptState(appState, appId);
    } else {
      LOG.info("Unknown child node with name: " + childNodeName);
    }
  }
}
 

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();

  Server server;
  try {
    secretMgr = new ClientToAMTokenSecretManager(
        this.appAttemptId, secretKey);
    server =
        new RPC.Builder(conf)
          .setProtocol(CustomProtocol.class)
          .setNumHandlers(1)
          .setSecretManager(secretMgr)
          .setInstance(this).build();
  } catch (Exception e) {
    throw new YarnRuntimeException(e);
  }
  server.start();
  this.address = NetUtils.getConnectAddress(server);
  super.serviceStart();
}
 

private void validateConf(Configuration conf) {
  // validate scheduler memory allocation setting
  int minMem = conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  int maxMem = conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);

  if (minMem <= 0 || minMem > maxMem) {
    throw new YarnRuntimeException("Invalid resource scheduler memory"
      + " allocation configuration"
      + ", " + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB
      + "=" + minMem
      + ", " + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB
      + "=" + maxMem + ", min and max should be greater than 0"
      + ", max should be no smaller than min.");
  }
}
 

public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 

private synchronized void loadRMAppState(RMState rmState) throws Exception {
  List<String> childNodes = getChildrenWithRetries(rmAppRoot, false);
  for (String childNodeName : childNodes) {
    String childNodePath = getNodePath(rmAppRoot, childNodeName);
    byte[] childData = getDataWithRetries(childNodePath, false);
    if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
      // application
      if (LOG.isDebugEnabled()) {
        LOG.debug("Loading application from znode: " + childNodeName);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
      ApplicationStateDataPBImpl appState =
          new ApplicationStateDataPBImpl(
              ApplicationStateDataProto.parseFrom(childData));
      if (!appId.equals(
          appState.getApplicationSubmissionContext().getApplicationId())) {
        throw new YarnRuntimeException("The child node name is different " +
            "from the application id");
      }
      rmState.appState.put(appId, appState);
      loadApplicationAttemptState(appState, appId);
    } else {
      LOG.info("Unknown child node with name: " + childNodeName);
    }
  }
}
 

protected void resyncWithRM() {
  //we do not want to block dispatcher thread here
  new Thread() {
    @Override
    public void run() {
      try {
        LOG.info("Notifying ContainerManager to block new container-requests");
        containerManager.setBlockNewContainerRequests(true);
        if (!rmWorkPreservingRestartEnabled) {
          LOG.info("Cleaning up running containers on resync");
          containerManager.cleanupContainersOnNMResync();
        } else {
          LOG.info("Preserving containers on resync");
        }
        ((NodeStatusUpdaterImpl) nodeStatusUpdater)
          .rebootNodeStatusUpdaterAndRegisterWithRM();
      } catch (YarnRuntimeException e) {
        LOG.fatal("Error while rebooting NodeStatusUpdater.", e);
        shutDown();
      }
    }
  }.start();
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  try {
    //Create the staging directory if it does not exist
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    Path stagingDir = MRApps.getStagingAreaDir(conf, user);
    FileSystem fs = getFileSystem(conf);
    fs.mkdirs(stagingDir);
  } catch (Exception e) {
    throw new YarnRuntimeException("Error creating staging dir", e);
  }
  
  super.serviceInit(conf);
  if (this.clusterInfo != null) {
    getContext().getClusterInfo().setMaxContainerCapability(
        this.clusterInfo.getMaxContainerCapability());
  } else {
    getContext().getClusterInfo().setMaxContainerCapability(
        Resource.newInstance(10240, 1));
  }
}
 

public DelegationTokenToRenew(Collection<ApplicationId> applicationIds,
    Token<?> token,
    Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
    String user) {
  this.token = token;
  this.user = user;
  if (token.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) {
    try {
      AbstractDelegationTokenIdentifier identifier =
          (AbstractDelegationTokenIdentifier) token.decodeIdentifier();
      maxDate = identifier.getMaxDate();
    } catch (IOException e) {
      throw new YarnRuntimeException(e);
    }
  }
  this.referringAppIds = Collections.synchronizedSet(
      new HashSet<ApplicationId>(applicationIds));
  this.conf = conf;
  this.expirationDate = expirationDate;
  this.timerTask = null;
  this.shouldCancelAtEnd = shouldCancelAtEnd;
}
 

@Test (timeout = 30000)
public void testConfValidation() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
  try {
    scheduler.serviceInit(conf);
    fail("Exception is expected because the min memory allocation is" +
      " larger than the max memory allocation.");
  } catch (YarnRuntimeException e) {
    // Exception is expected.
    assertTrue("The thrown exception is not the expected one.",
      e.getMessage().startsWith(
        "Invalid resource scheduler memory"));
  }
}
 

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  MRClientProtocol instance = new MRClientProtocolTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
        MRClientProtocol.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete server");
  } finally {
    server.stop();
  }
}
 

public List<TaskAttemptContainerAssignedEvent> schedule()
    throws Exception {
  // before doing heartbeat with RM, drain all the outstanding events to
  // ensure all the requests before this heartbeat is to be handled
  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      return eventQueue.isEmpty();
    }
  }, 100, 10000);
  // run the scheduler
  try {
    super.heartbeat();
  } catch (Exception e) {
    LOG.error("error in heartbeat ", e);
    throw new YarnRuntimeException(e);
  }

  List<TaskAttemptContainerAssignedEvent> result
    = new ArrayList<TaskAttemptContainerAssignedEvent>(events);
  events.clear();
  return result;
}
 

@Test
public void testValidPathsDirHandlerService() throws Exception {
  Configuration conf = new YarnConfiguration();
  String localDir1 = new File("file:///" + testDir, "localDir1").getPath();
  String localDir2 = new File("hdfs:///" + testDir, "localDir2").getPath();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
  String logDir1 = new File("file:///" + testDir, "logDir1").getPath();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1);
  LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
  try {
    dirSvc.init(conf);
    Assert.fail("Service should have thrown an exception due to wrong URI");
  } catch (YarnRuntimeException e) {
  }
  Assert.assertEquals("Service should not be inited",
                      STATE.STOPPED,
                      dirSvc.getServiceState());
  dirSvc.close();
}
 

protected void resyncWithRM() {
  //we do not want to block dispatcher thread here
  new Thread() {
    @Override
    public void run() {
      try {
        LOG.info("Notifying ContainerManager to block new container-requests");
        containerManager.setBlockNewContainerRequests(true);
        if (!rmWorkPreservingRestartEnabled) {
          LOG.info("Cleaning up running containers on resync");
          containerManager.cleanupContainersOnNMResync();
        } else {
          LOG.info("Preserving containers on resync");
        }
        ((NodeStatusUpdaterImpl) nodeStatusUpdater)
          .rebootNodeStatusUpdaterAndRegisterWithRM();
      } catch (YarnRuntimeException e) {
        LOG.fatal("Error while rebooting NodeStatusUpdater.", e);
        shutDown();
      }
    }
  }.start();
}
 

private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  LocalizationProtocol instance = new LocalizationProtocolTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
          LocalizationProtocol.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to create server");
  } finally {
    if (server != null) {
      server.stop();
    }
  }
}
 

protected Planner getReplanner(String planQueueName) {
  ReservationSchedulerConfiguration reservationConfig =
      getReservationSchedulerConfiguration();
  String plannerClassName = reservationConfig.getReplanner(planQueueName);
  LOG.info("Using Replanner: " + plannerClassName + " for queue: "
      + planQueueName);
  try {
    Class<?> plannerClazz = conf.getClassByName(plannerClassName);
    if (Planner.class.isAssignableFrom(plannerClazz)) {
      Planner planner =
          (Planner) ReflectionUtils.newInstance(plannerClazz, conf);
      planner.init(planQueueName, reservationConfig);
      return planner;
    } else {
      throw new YarnRuntimeException("Class: " + plannerClazz
          + " not instance of " + Planner.class.getCanonicalName());
    }
  } catch (ClassNotFoundException e) {
    throw new YarnRuntimeException("Could not instantiate Planner: "
        + plannerClassName + " for queue: " + planQueueName, e);
  }
}
 

@Test
@SuppressWarnings("unchecked") // mocked generics
public void testContainerLocalizerClosesFilesystems() throws Exception {
  // verify filesystems are closed when localizer doesn't fail
  FileContext fs = FileContext.getLocalFSFileContext();
  spylfs = spy(fs.getDefaultFileSystem());
  ContainerLocalizer localizer = setupContainerLocalizerForTest();
  doNothing().when(localizer).localizeFiles(any(LocalizationProtocol.class),
      any(CompletionService.class), any(UserGroupInformation.class));
  verify(localizer, never()).closeFileSystems(
      any(UserGroupInformation.class));
  localizer.runLocalization(nmAddr);
  verify(localizer).closeFileSystems(any(UserGroupInformation.class));

  spylfs = spy(fs.getDefaultFileSystem());
  // verify filesystems are closed when localizer fails
  localizer = setupContainerLocalizerForTest();
  doThrow(new YarnRuntimeException("Forced Failure")).when(localizer).localizeFiles(
      any(LocalizationProtocol.class), any(CompletionService.class),
      any(UserGroupInformation.class));
  verify(localizer, never()).closeFileSystems(
      any(UserGroupInformation.class));
  localizer.runLocalization(nmAddr);
  verify(localizer).closeFileSystems(any(UserGroupInformation.class));
}
 

@Test
public void testContainerExecutorInitCall() {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      InvalidContainerExecutor.class,
      ContainerExecutor.class);
  try {
    nm.init(conf);
    fail("Init should fail");
  } catch (YarnRuntimeException e) {
    //PASS
    assert(e.getCause().getMessage().contains("dummy executor init called"));
  } finally {
    nm.stop();
  }
}