org.apache.hadoop.yarn.exceptions.YarnException Java Examples

@Override
public Map<ApplicationId, ApplicationReport> getAllApplications()
    throws YarnException, IOException {
  TimelineEntities entities = timelineDataManager.getEntities(
      ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ApplicationId, ApplicationReport> apps =
      new LinkedHashMap<ApplicationId, ApplicationReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      try {
        ApplicationReportExt app =
            generateApplicationReport(entity, ApplicationReportField.ALL);
        apps.put(app.appReport.getApplicationId(), app.appReport);
      } catch (Exception e) {
        LOG.error("Error on generating application report for " +
            entity.getEntityId(), e);
      }
    }
  }
  return apps;
}
 

@Override
public boolean isFinished()
{
  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport != null) {
      if (appReport.getFinalApplicationStatus() == null
          || appReport.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) {
        return false;
      }
    }
    return true;
  } catch (YarnException | IOException e) {
    throw Throwables.propagate(e);
  }
}
 

@Override
public JobStatus getTrainingJobStatus(String jobName)
    throws IOException, YarnException {
  if (this.serviceClient == null) {
    synchronized (this) {
      if (this.serviceClient == null) {
        this.serviceClient = YarnServiceUtils.createServiceClient(
            clientContext.getYarnConfig());
      }
    }
  }
  String appStatus = serviceClient.getStatusString(jobName);
  Service serviceSpec = ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
  JobStatus jobStatus = JobStatusBuilder.fromServiceSpec(serviceSpec);
  return jobStatus;
}
 

private ContainerReport getContainer(
    ContainerId containerId,
    HashMap<ApplicationAttemptId, List<ContainerReport>> containersToAppAttemptMapping)
    throws YarnException, IOException {
  List<ContainerReport> containersForAppAttempt =
      containersToAppAttemptMapping.get(containerId
          .getApplicationAttemptId());
  if (containersForAppAttempt == null) {
    throw new ApplicationNotFoundException(containerId
        .getApplicationAttemptId().getApplicationId() + " is not found ");
  }
  Iterator<ContainerReport> iterator = containersForAppAttempt.iterator();
  while (iterator.hasNext()) {
    ContainerReport next = iterator.next();
    if (next.getContainerId().equals(containerId)) {
      return next;
    }
  }
  throw new ContainerNotFoundException(containerId + " is not found ");
}
 

@Test
public void testSubmitReservationEmptyRR() {
  ReservationSubmissionRequest request =
      createSimpleReservationSubmissionRequest(1, 0, 1, 5, 3);
  Plan plan = null;
  try {
    plan =
        rrValidator.validateReservationSubmissionRequest(rSystem, request,
            ReservationSystemTestUtil.getNewReservationId());
    Assert.fail();
  } catch (YarnException e) {
    Assert.assertNull(plan);
    String message = e.getMessage();
    Assert.assertTrue(message
        .startsWith("No resources have been specified to reserve"));
    LOG.info(message);
  }
}
 

@Override
public RegisterNodeManagerResponse registerNodeManager(
    RegisterNodeManagerRequest request) throws YarnException,
    IOException {
  NodeId nodeId = request.getNodeId();
  Resource resource = request.getResource();
  LOG.info("Registering " + nodeId.toString());
  // NOTE: this really should be checking against the config value
  InetSocketAddress expected = NetUtils.getConnectAddress(
      conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
  Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString());
  Assert.assertEquals(5 * 1024, resource.getMemory());
  registeredNodes.add(nodeId);

  RegisterNodeManagerResponse response = recordFactory
      .newRecordInstance(RegisterNodeManagerResponse.class);
  response.setContainerTokenMasterKey(createMasterKey());
  response.setNMTokenMasterKey(createMasterKey());
  return response;
}
 

@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
    RefreshUserToGroupsMappingsRequest request)
    throws YarnException, IOException {
  String argName = "refreshUserToGroupsMappings";
  UserGroupInformation user = checkAcls(argName);

  checkRMStatus(user.getShortUserName(), argName, "refresh user-groups.");

  Groups.getUserToGroupsMappingService(
      getConfiguration(new Configuration(false),
          YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();

  RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");

  return recordFactory.newRecordInstance(
      RefreshUserToGroupsMappingsResponse.class);
}
 

@Test(timeout = 10000)
public void testGetApplications() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports =
      ((MockAHSClient) client).getReports();

  List<ApplicationReport> reports = client.getApplications();
  Assert.assertEquals(reports, expectedReports);

  reports = client.getApplications();
  Assert.assertEquals(reports.size(), 4);
  client.stop();
}
 

/**
 * {@inheritDoc}
 */
@Override
public ApplicationId submitJob(Parameter paramsHolder)
    throws IOException, YarnException {
  Framework framework = paramsHolder.getFramework();
  RunJobParameters parameters =
      (RunJobParameters) paramsHolder.getParameters();

  if (framework == Framework.TENSORFLOW) {
    return submitTensorFlowJob((TensorFlowRunJobParameters) parameters);
  } else if (framework == Framework.PYTORCH) {
    return submitPyTorchJob((PyTorchRunJobParameters) parameters);
  } else {
    throw new UnsupportedOperationException(SUPPORTED_FRAMEWORKS_MESSAGE);
  }
}
 

@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 2)));
  client.stop();
}
 

@Override
public RegisterNodeManagerResponse registerNodeManager(
    RegisterNodeManagerRequest request) throws YarnException,
    IOException {
  NodeId nodeId = request.getNodeId();
  Resource resource = request.getResource();
  LOG.info("Registering " + nodeId.toString());
  // NOTE: this really should be checking against the config value
  InetSocketAddress expected = NetUtils.getConnectAddress(
      conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
  Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString());
  Assert.assertEquals(5 * 1024, resource.getMemory());
  registeredNodes.add(nodeId);

  RegisterNodeManagerResponse response = recordFactory
      .newRecordInstance(RegisterNodeManagerResponse.class);
  response.setContainerTokenMasterKey(createMasterKey());
  response.setNMTokenMasterKey(createMasterKey());
  return response;
}
 

/**
 * Clean all resource references to a cache resource that contain application
 * ids pointing to finished applications. If the resource key does not exist,
 * do nothing.
 *
 * @param key a unique identifier for a resource
 * @throws YarnException
 */
@Private
public void cleanResourceReferences(String key) throws YarnException {
  Collection<SharedCacheResourceReference> refs = getResourceReferences(key);
  if (!refs.isEmpty()) {
    Set<SharedCacheResourceReference> refsToRemove =
        new HashSet<SharedCacheResourceReference>();
    for (SharedCacheResourceReference r : refs) {
      if (!appChecker.isApplicationActive(r.getAppId())) {
        // application in resource reference is dead, it is safe to remove the
        // reference
        refsToRemove.add(r);
      }
    }
    if (refsToRemove.size() > 0) {
      removeResourceReferences(key, refsToRemove, false);
    }
  }
}
 

@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws YarnException {
  try {
    if (!isAllowedDelegationTokenOp()) {
      throw new IOException(
          "Delegation Token can be renewed only with kerberos authentication");
    }
    
    org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
    Token<RMDelegationTokenIdentifier> token = new Token<RMDelegationTokenIdentifier>(
        protoToken.getIdentifier().array(), protoToken.getPassword().array(),
        new Text(protoToken.getKind()), new Text(protoToken.getService()));

    String user = getRenewerForToken(token);
    long nextExpTime = rmDTSecretManager.renewToken(token, user);
    RenewDelegationTokenResponse renewResponse = Records
        .newRecord(RenewDelegationTokenResponse.class);
    renewResponse.setNextExpirationTime(nextExpTime);
    return renewResponse;
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }
}
 

private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {
	List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

	int totalFreeMemory = 0;
	int containerLimit = 0;
	int[] nodeManagersFree = new int[nodes.size()];

	for (int i = 0; i < nodes.size(); i++) {
		NodeReport rep = nodes.get(i);
		int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0);
		nodeManagersFree[i] = free;
		totalFreeMemory += free;
		if (free > containerLimit) {
			containerLimit = free;
		}
	}
	return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree);
}
 

/**
 * Test the method registerNodeManager. Method should return a not null
 * result.
 * 
 */
@Test
public void testResourceTrackerPBClientImpl() throws Exception {
  RegisterNodeManagerRequest request = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  assertNotNull(client.registerNodeManager(request));
  
  ResourceTrackerTestImpl.exception = true;
  try {
    client.registerNodeManager(request);
    fail("there  should be YarnException");
  } catch (YarnException e) {
    assertTrue(e.getMessage().startsWith("testMessage"));
  }finally{
    ResourceTrackerTestImpl.exception = false;
  }

}
 

@Test
public void testDeleteReservationNoID() {
  ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
  Plan plan = null;
  try {
    plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
    Assert.fail();
  } catch (YarnException e) {
    Assert.assertNull(plan);
    String message = e.getMessage();
    Assert
        .assertTrue(message
            .startsWith("Missing reservation id. Please try again by specifying a reservation id."));
    LOG.info(message);
  }
}
 

@Override
public synchronized void killAllAppsInQueue(String queueName)
    throws YarnException {
  // check if queue is a valid
  List<ApplicationAttemptId> apps = getAppsInQueue(queueName);
  if (apps == null) {
    String errMsg = "The specified Queue: " + queueName + " doesn't exist";
    LOG.warn(errMsg);
    throw new YarnException(errMsg);
  }
  // generate kill events for each pending/running app
  for (ApplicationAttemptId app : apps) {
    this.rmContext
        .getDispatcher()
        .getEventHandler()
        .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
  }
}
 

/**
 * Add or update an domain. If the domain already exists, only the owner
 * and the admin can update it.
 */
public void putDomain(TimelineDomain domain,
    UserGroupInformation callerUGI) throws YarnException, IOException {
  TimelineDomain existingDomain =
      store.getDomain(domain.getId());
  if (existingDomain != null) {
    if (!timelineACLsManager.checkAccess(callerUGI, existingDomain)) {
      throw new YarnException(callerUGI.getShortUserName() +
          " is not allowed to override an existing domain " +
          existingDomain.getId());
    }
    // Set it again in case ACLs are not enabled: The domain can be
    // modified by every body, but the owner is not changed.
    domain.setOwner(existingDomain.getOwner());
  }
  store.put(domain);
  // If the domain exists already, it is likely to be in the cache.
  // We need to invalidate it.
  if (existingDomain != null) {
    timelineACLsManager.replaceIfExist(domain);
  }
}
 

private void checkAcls(String method) throws YarnException {
  UserGroupInformation user;
  try {
    user = UserGroupInformation.getCurrentUser();
  } catch (IOException ioe) {
    LOG.warn("Couldn't get current user", ioe);
    throw RPCUtil.getRemoteException(ioe);
  }

  if (!authorizer.isAdmin(user)) {
    LOG.warn("User " + user.getShortUserName() + " doesn't have permission" +
        " to call '" + method + "'");

    throw RPCUtil.getRemoteException(
        new AccessControlException("User " + user.getShortUserName() +
        " doesn't have permission" + " to call '" + method + "'"));
  }
  LOG.info("SCM Admin: " + method + " invoked by user " +
      user.getShortUserName());
}
 

@Override
public SubmitApplicationResponse submitApplication(
    SubmitApplicationRequest request) throws YarnException,
    IOException {
  SubmitApplicationRequestProto requestProto =
      ((SubmitApplicationRequestPBImpl) request).getProto();
  try {
    return new SubmitApplicationResponsePBImpl(proxy.submitApplication(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 

@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  return RenewDelegationTokenResponse
      .newInstance(createNextExpirationTime());
}
 

@Override
public GetContainersResponse getContainers(GetContainersRequest request)
    throws YarnException, IOException {
  GetContainersRequestProto requestProto =
      ((GetContainersRequestPBImpl) request).getProto();
  try {
    return new GetContainersResponsePBImpl(proxy.getContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public List<QueueInfo> getAllQueues() throws YarnException,
    IOException {
  List<QueueInfo> queues = new ArrayList<QueueInfo>();

  QueueInfo rootQueue =
      rmClient.getQueueInfo(getQueueInfoRequest(ROOT, false, true, true))
        .getQueueInfo();
  getChildQueues(rootQueue, queues, true);
  return queues;
}
 

public Map<String, Long> getJobsDuration(RMCommunicator rmCommunicator) throws YarnException, IOException {

		long appDuration, endMillis = 0;
		Map<String, Long> jobsDuration = new HashMap<String, Long>();
		for (ApplicationReport report : rmCommunicator.getApplications()) {
			if (report.getFinalApplicationStatus().equals(FinalApplicationStatus.SUCCEEDED)) {
				endMillis = report.getFinishTime();
				appDuration = (endMillis - report.getStartTime());
				jobsDuration.put(report.getApplicationId().toString().replace(APPLICATION, JOB), appDuration);
			}
		}	
		return jobsDuration;
	}
 

/**
 * To ensure there are not multiple instances of the SCM running on a given
 * cluster, a global pid file is used. This file contains the hostname of the
 * machine that owns the pid file.
 *
 * @return true if the pid file was written, false otherwise
 * @throws YarnException
 */
private boolean writeGlobalCleanerPidFile() throws YarnException {
  String root =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  Path pidPath = new Path(root, GLOBAL_CLEANER_PID);
  try {
    FileSystem fs = FileSystem.get(this.conf);

    if (fs.exists(pidPath)) {
      return false;
    }

    FSDataOutputStream os = fs.create(pidPath, false);
    // write the hostname and the process id in the global cleaner pid file
    final String ID = ManagementFactory.getRuntimeMXBean().getName();
    os.writeUTF(ID);
    os.close();
    // add it to the delete-on-exit to ensure it gets deleted when the JVM
    // exits
    fs.deleteOnExit(pidPath);
  } catch (IOException e) {
    throw new YarnException(e);
  }
  LOG.info("Created the global cleaner pid file at " + pidPath.toString());
  return true;
}
 

@Test(timeout=10000)
public void testSerializedExceptionDeSer() throws Exception{
  // without cause
  YarnException yarnEx = new YarnException("Yarn_Exception");
  SerializedException serEx = SerializedException.newInstance(yarnEx);
  Throwable throwable = serEx.deSerialize();
  Assert.assertEquals(yarnEx.getClass(), throwable.getClass());
  Assert.assertEquals(yarnEx.getMessage(), throwable.getMessage());

  // with cause
  IOException ioe = new IOException("Test_IOException");
  RuntimeException runtimeException =
      new RuntimeException("Test_RuntimeException", ioe);
  YarnException yarnEx2 =
      new YarnException("Test_YarnException", runtimeException);

  SerializedException serEx2 = SerializedException.newInstance(yarnEx2);
  Throwable throwable2 = serEx2.deSerialize();
  throwable2.printStackTrace();
  Assert.assertEquals(yarnEx2.getClass(), throwable2.getClass());
  Assert.assertEquals(yarnEx2.getMessage(), throwable2.getMessage());

  Assert.assertEquals(runtimeException.getClass(), throwable2.getCause().getClass());
  Assert.assertEquals(runtimeException.getMessage(), throwable2.getCause().getMessage());

  Assert.assertEquals(ioe.getClass(), throwable2.getCause().getCause().getClass());
  Assert.assertEquals(ioe.getMessage(), throwable2.getCause().getCause().getMessage());
}
 

@Override
public GetNewApplicationResponse getNewApplication(
    GetNewApplicationRequest request) throws YarnException,
    IOException {
  GetNewApplicationRequestProto requestProto =
      ((GetNewApplicationRequestPBImpl) request).getProto();
  try {
    return new GetNewApplicationResponsePBImpl(proxy.getNewApplication(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public KillApplicationResponse forceKillApplication(
    KillApplicationRequest request) throws YarnException {
  ApplicationId applicationId = request.getApplicationId();
  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application.isAppFinalStateStored()) {
    return KillApplicationResponse.newInstance(true);
  } else {
    return KillApplicationResponse.newInstance(false);
  }
}
 

private void stopAllExistingMRJobs(String blurEnv, Configuration conf) throws YarnException, IOException,
    InterruptedException {
  Cluster cluster = new Cluster(conf);
  JobStatus[] allJobStatuses = cluster.getAllJobStatuses();
  for (JobStatus jobStatus : allJobStatuses) {
    if (jobStatus.isJobComplete()) {
      continue;
    }
    String jobFile = jobStatus.getJobFile();
    JobID jobID = jobStatus.getJobID();
    Job job = cluster.getJob(jobID);
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    Configuration configuration = new Configuration(false);
    Path path = new Path(jobFile);
    Path makeQualified = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
    if (hasReadAccess(fileSystem, makeQualified)) {
      try (FSDataInputStream in = fileSystem.open(makeQualified)) {
        configuration.addResource(copy(in));
      }
      String jobBlurEnv = configuration.get(BLUR_ENV);
      LOG.info("Checking job [{0}] has env [{1}] current env set to [{2}]", jobID, jobBlurEnv, blurEnv);
      if (blurEnv.equals(jobBlurEnv)) {
        LOG.info("Killing running job [{0}]", jobID);
        job.killJob();
      }
    }
  }
}