org.apache.hadoop.classification.InterfaceAudience.Private Java Examples

/**
 * Clean all resource references to a cache resource that contain application
 * ids pointing to finished applications. If the resource key does not exist,
 * do nothing.
 *
 * @param key a unique identifier for a resource
 * @throws YarnException
 */
@Private
public void cleanResourceReferences(String key) throws YarnException {
  Collection<SharedCacheResourceReference> refs = getResourceReferences(key);
  if (!refs.isEmpty()) {
    Set<SharedCacheResourceReference> refsToRemove =
        new HashSet<SharedCacheResourceReference>();
    for (SharedCacheResourceReference r : refs) {
      if (!appChecker.isApplicationActive(r.getAppId())) {
        // application in resource reference is dead, it is safe to remove the
        // reference
        refsToRemove.add(r);
      }
    }
    if (refsToRemove.size() > 0) {
      removeResourceReferences(key, refsToRemove, false);
    }
  }
}
 

/**
 * Create an instance of the AppChecker service via reflection based on the
 * {@link YarnConfiguration#SCM_APP_CHECKER_CLASS} parameter.
 * 
 * @param conf
 * @return an instance of the AppChecker class
 */
@Private
@SuppressWarnings("unchecked")
public static AppChecker createAppCheckerService(Configuration conf) {
  Class<? extends AppChecker> defaultCheckerClass;
  try {
    defaultCheckerClass =
        (Class<? extends AppChecker>) Class
            .forName(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
  } catch (Exception e) {
    throw new YarnRuntimeException("Invalid default scm app checker class"
        + YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS, e);
  }

  AppChecker checker =
      ReflectionUtils.newInstance(conf.getClass(
          YarnConfiguration.SCM_APP_CHECKER_CLASS, defaultCheckerClass,
          AppChecker.class), conf);
  return checker;
}
 

/**
 * Given the {@link Configuration} and {@link ACL}s used (zkAcl) for
 * ZooKeeper access, construct the {@link ACL}s for the store's root node.
 * In the constructed {@link ACL}, all the users allowed by zkAcl are given
 * rwa access, while the current RM has exclude create-delete access.
 *
 * To be called only when HA is enabled and the configuration doesn't set ACL
 * for the root node.
 */
@VisibleForTesting
@Private
@Unstable
protected List<ACL> constructZkRootNodeACL(
    Configuration conf, List<ACL> sourceACLs) throws NoSuchAlgorithmException {
  List<ACL> zkRootNodeAcl = new ArrayList<ACL>();
  for (ACL acl : sourceACLs) {
    zkRootNodeAcl.add(new ACL(
        ZKUtil.removeSpecificPerms(acl.getPerms(), CREATE_DELETE_PERMS),
        acl.getId()));
  }

  zkRootNodeUsername = HAUtil.getConfValueForRMInstance(
      YarnConfiguration.RM_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
  Id rmId = new Id(zkRootNodeAuthScheme,
      DigestAuthenticationProvider.generateDigest(
          zkRootNodeUsername + ":" + zkRootNodePassword));
  zkRootNodeAcl.add(new ACL(CREATE_DELETE_PERMS, rmId));
  return zkRootNodeAcl;
}
 

@Private
public static String getCacheEntryPath(int cacheDepth, String cacheRoot,
    String checksum) {

  if (cacheDepth <= 0) {
    throw new IllegalArgumentException(
        "The cache depth must be greater than 0. Passed value: " + cacheDepth);
  }
  if (checksum.length() < cacheDepth) {
    throw new IllegalArgumentException("The checksum passed was too short: "
        + checksum);
  }

  // Build the cache entry path to the specified depth. For example, if the
  // depth is 3 and the checksum is 3c4f, the path would be:
  // SHARED_CACHE_ROOT/3/c/4/3c4f
  StringBuilder sb = new StringBuilder(cacheRoot);
  for (int i = 0; i < cacheDepth; i++) {
    sb.append(Path.SEPARATOR_CHAR);
    sb.append(checksum.charAt(i));
  }
  sb.append(Path.SEPARATOR_CHAR).append(checksum);

  return sb.toString();
}
 

@Private
@Unstable
public static ReservationUpdateResponse newInstance() {
  ReservationUpdateResponse response =
      Records.newRecord(ReservationUpdateResponse.class);
  return response;
}
 

@Private
@Unstable
public static SubmitApplicationResponse newInstance() {
  SubmitApplicationResponse response =
      Records.newRecord(SubmitApplicationResponse.class);
  return response;
}
 

@Private
@Unstable
public static RefreshUserToGroupsMappingsResponse newInstance() {
  RefreshUserToGroupsMappingsResponse response =
      Records.newRecord(RefreshUserToGroupsMappingsResponse.class);
  return response;
}
 

@Private
@Override
protected void checkAllowedProtocols(Class<?> protocol) {
  Preconditions.checkArgument(
      protocol.isAssignableFrom(ClientRMProtocols.class),
      "RM does not support this client protocol");
}
 

@Private
@Unstable
public static ContainerStatus newInstance(ContainerId containerId,
    ContainerState containerState, String diagnostics, int exitStatus) {
  ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
  containerStatus.setState(containerState);
  containerStatus.setContainerId(containerId);
  containerStatus.setDiagnostics(diagnostics);
  containerStatus.setExitStatus(exitStatus);
  return containerStatus;
}
 

@Private
@VisibleForTesting
public MasterKeyData getNextMasterKeyData() {
  this.readLock.lock();
  try {
    return this.nextMasterKey;
  } finally {
    this.readLock.unlock();
  }
}
 

@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
    ApplicationAttemptId applicationAttemptId, String user, String queue,
    String name, String host, int rpcPort, Token clientToAMToken,
    YarnApplicationState state, String diagnostics, String url,
    long startTime, long finishTime, FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String applicationType, Token amRmToken) {
  ApplicationReport report = Records.newRecord(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(applicationType);
  report.setAMRMToken(amRmToken);
  return report;
}
 

@Private
@Unstable
public static ApplicationId newInstance(long clusterTimestamp, int id) {
  ApplicationId appId = Records.newRecord(ApplicationId.class);
  appId.setClusterTimestamp(clusterTimestamp);
  appId.setId(id);
  appId.build();
  return appId;
}
 

@Private
@VisibleForTesting
protected AMRMClientAsync(AMRMClient<T> client, int intervalMs,
    CallbackHandler callbackHandler) {
  super(AMRMClientAsync.class.getName());
  this.client = client;
  this.heartbeatIntervalMs.set(intervalMs);
  this.handler = callbackHandler;
}
 

/**
 * Get the ResourceManager address from the provided Configuration for the
 * given protocol.
 */
@Private
protected InetSocketAddress getRMAddress(
    YarnConfiguration conf, Class<?> protocol) throws IOException {
  throw new UnsupportedOperationException("This method should be invoked " +
      "from an instance of ClientRMProxy or ServerRMProxy");
}
 

/**
 * Get a proxy to the RM at the specified address. To be used to create a
 * RetryProxy.
 */
@Private
static <T> T getProxy(final Configuration conf,
    final Class<T> protocol, final InetSocketAddress rmAddress)
    throws IOException {
  return UserGroupInformation.getCurrentUser().doAs(
    new PrivilegedAction<T>() {
      @Override
      public T run() {
        return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf);
      }
    });
}
 

@Private
public ApplicationAttemptId getApplicationAttemptId() {
  if (!proto.hasAppAttemptId()) {
    return null;
  }
  return new ApplicationAttemptIdPBImpl(proto.getAppAttemptId());
}
 

@Private
@Unstable
public ContainerContext(String user, ContainerId containerId,
    Resource resource) {
  this.user = user;
  this.containerId = containerId;
  this.resource = resource;
}
 

@Private
public Resource getResourceLimit() {
  Resource headRoom = getAvailableResources();
  if (headRoom == null) {
    headRoom = Resources.none();
  }
  Resource assignedMapResource =
      Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
  Resource assignedReduceResource =
      Resources.multiply(reduceResourceRequest,
        assignedRequests.reduces.size());
  return Resources.add(headRoom,
    Resources.add(assignedMapResource, assignedReduceResource));
}
 

@Private
@Unstable
public static RefreshNodesResponse newInstance() {
  RefreshNodesResponse response =
      Records.newRecord(RefreshNodesResponse.class);
  return response;
}
 

@Override
@Private
public Token<?> getDelegationToken(String renewer) throws IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return underlyingFs.getDelegationToken(renewer);
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}
 

@Private
@VisibleForTesting
protected void populateNMTokens(List<NMToken> nmTokens) {
  for (NMToken token : nmTokens) {
    String nodeId = token.getNodeId().toString();
    if (getNMTokenCache().containsToken(nodeId)) {
      LOG.info("Replacing token for : " + nodeId);
    } else {
      LOG.info("Received new token for : " + nodeId);
    }
    getNMTokenCache().setToken(nodeId, token.getToken());
  }
}
 

@Private
public HostsFileReader(String includesFile, InputStream inFileInputStream,
    String excludesFile, InputStream exFileInputStream) throws IOException {
  includes = new HashSet<String>();
  excludes = new HashSet<String>();
  this.includesFile = includesFile;
  this.excludesFile = excludesFile;
  refresh(inFileInputStream, exFileInputStream);
}
 

@Private
@Unstable
public static YarnClusterMetrics newInstance(int numNodeManagers) {
  YarnClusterMetrics metrics = Records.newRecord(YarnClusterMetrics.class);
  metrics.setNumNodeManagers(numNodeManagers);
  return metrics;
}
 

@Private
@Unstable
public static GetNewApplicationResponse newInstance(
    ApplicationId applicationId, Resource minCapability,
    Resource maxCapability) {
  GetNewApplicationResponse response =
      Records.newRecord(GetNewApplicationResponse.class);
  response.setApplicationId(applicationId);
  response.setMaximumResourceCapability(maxCapability);
  return response;
}
 

@Private
@Unstable
public static GetApplicationReportResponse newInstance(
    ApplicationReport ApplicationReport) {
  GetApplicationReportResponse response =
      Records.newRecord(GetApplicationReportResponse.class);
  response.setApplicationReport(ApplicationReport);
  return response;
}
 

@Private
@Unstable
public static StopContainersResponse newInstance(
    List<ContainerId> succeededRequests,
    Map<ContainerId, SerializedException> failedRequests) {
  StopContainersResponse response =
      Records.newRecord(StopContainersResponse.class);
  response.setFailedRequests(failedRequests);
  response.setSuccessfullyStoppedContainers(succeededRequests);
  return response;
}
 

@Private
@Unstable
public static CancelDelegationTokenResponse newInstance() {
  CancelDelegationTokenResponse response =
      Records.newRecord(CancelDelegationTokenResponse.class);
  return response;
}
 

@Private
@Unstable
public static GetQueueInfoResponse newInstance(QueueInfo queueInfo) {
  GetQueueInfoResponse response = Records.newRecord(GetQueueInfoResponse.class);
  response.setQueueInfo(queueInfo);
  return response;
}
 

@Private
@Unstable
public static GetDelegationTokenResponse newInstance(Token rmDTToken) {
  GetDelegationTokenResponse response =
      Records.newRecord(GetDelegationTokenResponse.class);
  response.setRMDelegationToken(rmDTToken);
  return response;
}
 

@Private
@Unstable
public abstract void setName(String name);