org.apache.hadoop.yarn.security.ContainerTokenIdentifier Java Examples

private LogAggregationContext getLogAggregationContextFromContainerToken(
    MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext)
    throws Exception {
  RMApp app2 = rm1.submitApp(200, logAggregationContext);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1.nodeHeartbeat(true);
  // request a container.
  am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>());
  ContainerId containerId =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED);

  // acquire the container.
  List<Container> containers =
      am2.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(containers.get(0)
        .getContainerToken());
  return token.getLogAggregationContext();
}
 

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 

/**
 * Container start has gone through. We need to store the containerId in order
 * to block future container start requests with same container token. This
 * container token needs to be saved till its container token expires.
 */
public synchronized void startContainerSuccessful(
    ContainerTokenIdentifier tokenId) {

  removeAnyContainerTokenIfExpired();
  
  ContainerId containerId = tokenId.getContainerID();
  Long expTime = tokenId.getExpiryTimeStamp();
  // We might have multiple containers with same expiration time.
  if (!recentlyStartedContainerTracker.containsKey(expTime)) {
    recentlyStartedContainerTracker
      .put(expTime, new ArrayList<ContainerId>());
  }
  recentlyStartedContainerTracker.get(expTime).add(containerId);
  try {
    stateStore.storeContainerToken(containerId, expTime);
  } catch (IOException e) {
    LOG.error("Unable to store token for container " + containerId, e);
  }
}
 

/**
 * Container will be remembered based on expiration time of the container
 * token used for starting the container. It is safe to use expiration time
 * as there is one to many mapping between expiration time and containerId.
 * @return true if the current token identifier is not present in cache.
 */
public synchronized boolean isValidStartContainerRequest(
    ContainerTokenIdentifier containerTokenIdentifier) {

  removeAnyContainerTokenIfExpired();

  Long expTime = containerTokenIdentifier.getExpiryTimeStamp();
  List<ContainerId> containers =
      this.recentlyStartedContainerTracker.get(expTime);
  if (containers == null
      || !containers.contains(containerTokenIdentifier.getContainerID())) {
    return true;
  } else {
    return false;
  }
}
 

protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
    org.apache.hadoop.yarn.api.records.Token token,
    ContainerTokenIdentifier containerTokenIdentifier) throws YarnException,
    InvalidToken {
  byte[] password =
      context.getContainerTokenSecretManager().retrievePassword(
        containerTokenIdentifier);
  byte[] tokenPass = token.getPassword().array();
  if (password == null || tokenPass == null
      || !Arrays.equals(password, tokenPass)) {
    throw new InvalidToken(
      "Invalid container token used for starting container on : "
          + context.getNodeId().toString());
  }
  return containerTokenIdentifier;
}
 

public ContainerImpl(Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  stateMachine = stateMachineFactory.make(this);
}
 

private LogAggregationContext getLogAggregationContextFromContainerToken(
    MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext)
    throws Exception {
  RMApp app2 = rm1.submitApp(200, logAggregationContext);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1.nodeHeartbeat(true);
  // request a container.
  am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>());
  ContainerId containerId =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED);

  // acquire the container.
  List<Container> containers =
      am2.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(containers.get(0)
        .getContainerToken());
  return token.getLogAggregationContext();
}
 

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 

/**
 * Container start has gone through. We need to store the containerId in order
 * to block future container start requests with same container token. This
 * container token needs to be saved till its container token expires.
 */
public synchronized void startContainerSuccessful(
    ContainerTokenIdentifier tokenId) {

  removeAnyContainerTokenIfExpired();
  
  ContainerId containerId = tokenId.getContainerID();
  Long expTime = tokenId.getExpiryTimeStamp();
  // We might have multiple containers with same expiration time.
  if (!recentlyStartedContainerTracker.containsKey(expTime)) {
    recentlyStartedContainerTracker
      .put(expTime, new ArrayList<ContainerId>());
  }
  recentlyStartedContainerTracker.get(expTime).add(containerId);
  try {
    stateStore.storeContainerToken(containerId, expTime);
  } catch (IOException e) {
    LOG.error("Unable to store token for container " + containerId, e);
  }
}
 

/**
 * Container will be remembered based on expiration time of the container
 * token used for starting the container. It is safe to use expiration time
 * as there is one to many mapping between expiration time and containerId.
 * @return true if the current token identifier is not present in cache.
 */
public synchronized boolean isValidStartContainerRequest(
    ContainerTokenIdentifier containerTokenIdentifier) {

  removeAnyContainerTokenIfExpired();

  Long expTime = containerTokenIdentifier.getExpiryTimeStamp();
  List<ContainerId> containers =
      this.recentlyStartedContainerTracker.get(expTime);
  if (containers == null
      || !containers.contains(containerTokenIdentifier.getContainerID())) {
    return true;
  } else {
    return false;
  }
}
 

protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
    org.apache.hadoop.yarn.api.records.Token token,
    ContainerTokenIdentifier containerTokenIdentifier) throws YarnException,
    InvalidToken {
  byte[] password =
      context.getContainerTokenSecretManager().retrievePassword(
        containerTokenIdentifier);
  byte[] tokenPass = token.getPassword().array();
  if (password == null || tokenPass == null
      || !Arrays.equals(password, tokenPass)) {
    throw new InvalidToken(
      "Invalid container token used for starting container on : "
          + context.getNodeId().toString());
  }
  return containerTokenIdentifier;
}
 

public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,Set<Integer> cpuCores) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.currentResource = resource;
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();
  this.cpuCores  = cpuCores;
  this.context = context;

  stateMachine = stateMachineFactory.make(this);
}
 

public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager,
    LogAggregationContext logAggregationContext)
    throws IOException {
  Resource r = BuilderUtils.newResource(1024, 1);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
        System.currentTimeMillis() + 100000L, 123, rmIdentifier,
        Priority.newInstance(0), 0, logAggregationContext);
  Token containerToken =
      BuilderUtils
        .newContainerToken(nodeId, containerTokenSecretManager
          .retrievePassword(containerTokenIdentifier),
          containerTokenIdentifier);
  return containerToken;
}
 

public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,
    RecoveredContainerStatus recoveredStatus, int exitCode,
    String diagnostics, boolean wasKilled, Set<Integer> cpuCores) {
  this(context,conf, dispatcher, stateStore, launchContext, creds, metrics,
      containerTokenIdentifier,cpuCores);
  this.recoveredStatus = recoveredStatus;
  this.exitCode = exitCode;
  this.recoveredAsKilled = wasKilled;
  this.diagnostics.append(diagnostics);
}
 

/**
 * Override of this is to validate ContainerTokens generated by using
 * different {@link MasterKey}s.
 */
@Override
public synchronized byte[] retrievePassword(
    ContainerTokenIdentifier identifier) throws SecretManager.InvalidToken {
  int keyId = identifier.getMasterKeyId();

  MasterKeyData masterKeyToUse = null;
  if (this.previousMasterKey != null
      && keyId == this.previousMasterKey.getMasterKey().getKeyId()) {
    // A container-launch has come in with a token generated off the last
    // master-key
    masterKeyToUse = this.previousMasterKey;
  } else if (keyId == super.currentMasterKey.getMasterKey().getKeyId()) {
    // A container-launch has come in with a token generated off the current
    // master-key
    masterKeyToUse = super.currentMasterKey;
  }

  if (nodeHostAddr != null
      && !identifier.getNmHostAddress().equals(nodeHostAddr)) {
    // Valid container token used for incorrect node.
    throw new SecretManager.InvalidToken("Given Container "
        + identifier.getContainerID().toString()
        + " identifier is not valid for current Node manager. Expected : "
        + nodeHostAddr + " Found : " + identifier.getNmHostAddress());
  }
  
  if (masterKeyToUse != null) {
    return retrievePasswordInternal(identifier, masterKeyToUse);
  }

  // Invalid request. Like startContainer() with token generated off
  // old-master-keys.
  throw new SecretManager.InvalidToken("Given Container "
      + identifier.getContainerID().toString()
      + " seems to have an illegally generated token.");
}
 

@Override
public byte[] createPassword(ContainerTokenIdentifier identifier) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Creating password for " + identifier.getContainerID()
        + " for user " + identifier.getUser() + " to be run on NM "
        + identifier.getNmHostAddress());
  }
  this.readLock.lock();
  try {
    return createPassword(identifier.getBytes(),
      this.currentMasterKey.getSecretKey());
  } finally {
    this.readLock.unlock();
  }
}
 

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 

@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());
  
  Set<Integer> cores= this.context.getCoresManager().allocateCores(containerId, 
  		                          token.getResource().getVirtualCores());
 
  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(this.context,getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled(),cores);
    
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
 

protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
    MasterKeyData masterKey)
    throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Retrieving password for " + identifier.getContainerID()
        + " for user " + identifier.getUser() + " to be run on NM "
        + identifier.getNmHostAddress());
  }
  return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
 

@Override
public byte[] retrievePassword(ContainerTokenIdentifier identifier)
    throws SecretManager.InvalidToken {
  this.readLock.lock();
  try {
    return retrievePasswordInternal(identifier, this.currentMasterKey);
  } finally {
    this.readLock.unlock();
  }
}
 

@Override
public byte[] createPassword(ContainerTokenIdentifier identifier) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Creating password for " + identifier.getContainerID()
        + " for user " + identifier.getUser() + " to be run on NM "
        + identifier.getNmHostAddress());
  }
  this.readLock.lock();
  try {
    return createPassword(identifier.getBytes(),
      this.currentMasterKey.getSecretKey());
  } finally {
    this.readLock.unlock();
  }
}
 

public static Token newContainerToken(NodeId nodeId, byte[] password,
    ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      Token.newInstance(tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}
 

public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}
 

protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
    MasterKeyData masterKey)
    throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Retrieving password for " + identifier.getContainerID()
        + " for user " + identifier.getUser() + " to be run on NM "
        + identifier.getNmHostAddress());
  }
  return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
 

protected ContainerTokenIdentifier waitForContainerTokenToExpire(
    ContainerTokenIdentifier identifier) {
  int attempts = 5;
  while (System.currentTimeMillis() < identifier.getExpiryTimeStamp()
      && attempts-- > 0) {
    try {
      Thread.sleep(1000);
    } catch (Exception e) {}
  }
  return identifier;
}
 

private Token createNewContainerToken(ContainerId contId,
    String containerManagerAddr) {
  long currentTime = System.currentTimeMillis();
  return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1",
      1234), "password".getBytes(), new ContainerTokenIdentifier(
      contId, containerManagerAddr, "user",
      Resource.newInstance(1024, 1, 1),
      currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0));
}
 

@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 

public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}
 

public static Token newContainerToken(NodeId nodeId,
    byte[] password, ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      Token.newInstance(tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}
 

public static Token newContainerToken(ContainerId cId, String host,
    int port, String user, Resource r, long expiryTime, int masterKeyId,
    byte[] password, long rmIdentifier) throws IOException {
  ContainerTokenIdentifier identifier =
      new ContainerTokenIdentifier(cId, host + ":" + port, user, r,
        expiryTime, masterKeyId, rmIdentifier, Priority.newInstance(0), 0);
  return newContainerToken(BuilderUtils.newNodeId(host, port), password,
      identifier);
}