org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier Java Examples

/**
 * From a token, get the session token identifier.
 *
 * @param token token to process
 * @return the session token identifier
 * @throws IOException failure to validate/read data encoded in identifier.
 * @throws IllegalArgumentException if the token isn't an GCP session token
 */
public static DelegationTokenIdentifier extractIdentifier(
    final Token<? extends DelegationTokenIdentifier> token) throws IOException {
  checkArgument(token != null, "null token");
  DelegationTokenIdentifier identifier;
  // harden up decode beyond what Token does itself
  try {
    identifier = token.decodeIdentifier();
  } catch (RuntimeException e) {
    Throwable cause = e.getCause();
    if (cause != null) {
      // its a wrapping around class instantiation.
      throw new DelegationTokenIOException("Decoding GCS token " + cause, cause);
    }
    throw e;
  }
  if (identifier == null) {
    throw new DelegationTokenIOException("Failed to unmarshall token " + token);
  }
  return identifier;
}
 

/**
 * Look up a token from the credentials, verify it is of the correct kind.
 *
 * @param credentials credentials to look up.
 * @param service service name
 * @param kind token kind to look for
 * @return the token or null if no suitable token was found
 * @throws DelegationTokenIOException wrong token kind found
 */
@SuppressWarnings("unchecked") // safe by contract of lookupToken()
private static Token<DelegationTokenIdentifier> lookupToken(
    Credentials credentials, Text service, Text kind) throws DelegationTokenIOException {
  logger.atFine().log("Looking for token for service %s in credentials", service);
  Token<?> token = credentials.getToken(service);
  if (token != null) {
    Text tokenKind = token.getKind();
    logger.atFine().log("Found token of kind %s", tokenKind);
    if (kind.equals(tokenKind)) {
      // The OAuth implementation catches and logs here; this one throws the failure up.
      return (Token<DelegationTokenIdentifier>) token;
    }

    // There's a token for this service, but it's not the right DT kind
    throw DelegationTokenIOException.tokenMismatch(service, kind, tokenKind);
  }
  // A token for the service was not found
  logger.atFine().log("No token found for %s", service);
  return null;
}
 

/**
 * Create a delegation token for the user. This will only be called if a new DT is needed, that
 * is: the filesystem has been deployed unbound.
 *
 * @return the token
 * @throws IOException if one cannot be created
 */
public Token<DelegationTokenIdentifier> createDelegationToken(String renewer) throws IOException {
  Text renewerText = new Text();
  if (renewer != null) {
    renewerText.set(renewer);
  }

  DelegationTokenIdentifier tokenIdentifier =
      requireNonNull(createTokenIdentifier(renewerText), "Token identifier");

  Token<DelegationTokenIdentifier> token = new Token<>(tokenIdentifier, secretManager);
  token.setKind(getKind());
  token.setService(service);
  logger.atFine().log("Created token %s with token identifier %s", token, tokenIdentifier);
  return token;
}
 

@SuppressWarnings("unchecked")
@Test
public void testRenewTokenSingleManager() throws Exception {
  for (int i = 0; i < TEST_RETRIES; i++) {
    DelegationTokenManager tm1 = null;
    String connectString = zkServer.getConnectString();
    Configuration conf = getSecretConf(connectString);
    tm1 = new DelegationTokenManager(conf, new Text("foo"));
    tm1.init();

    Token<DelegationTokenIdentifier> token =
        (Token<DelegationTokenIdentifier>)
        tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
    Assert.assertNotNull(token);
    tm1.renewToken(token, "foo");
    tm1.verifyToken(token);
    verifyDestroy(tm1, conf);
  }
}
 

@SuppressWarnings("unchecked")
@Test
public void testCancelTokenSingleManager() throws Exception {
  for (int i = 0; i < TEST_RETRIES; i++) {
    DelegationTokenManager tm1 = null;
    String connectString = zkServer.getConnectString();
    Configuration conf = getSecretConf(connectString);
    tm1 = new DelegationTokenManager(conf, new Text("foo"));
    tm1.init();

    Token<DelegationTokenIdentifier> token =
        (Token<DelegationTokenIdentifier>)
        tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
    Assert.assertNotNull(token);
    tm1.cancelToken(token, "foo");
    try {
      verifyTokenFail(tm1, token);
      fail("Expected InvalidToken");
    } catch (SecretManager.InvalidToken it) {
      it.printStackTrace();
    }
    verifyDestroy(tm1, conf);
  }
}
 

@Test
public void testAccessTokensProvidersValidation() throws IOException {
  GoogleHadoopFileSystem fs = new GoogleHadoopFileSystem();
  fs.initialize(new Path("gs://test/").toUri(), loadConfig());

  Token<?> dt = fs.getDelegationToken("current-user");
  assertThrows(
      "GCP Delegation tokens has already been bound/deployed",
      IllegalStateException.class,
      () -> fs.delegationTokens.bindToAnyDelegationToken());

  Token<DelegationTokenIdentifier> boundDT = fs.delegationTokens.getBoundDT();
  assertThrows(
      "GCP Delegation tokens has already been bound/deployed",
      IllegalStateException.class,
      () -> fs.delegationTokens.bindToDelegationToken(boundDT));

  Token<?> dt1 = fs.getDelegationToken("current-user");
  assertWithMessage("Tokens should be the same").that(dt1).isEqualTo(dt);
}
 

@SuppressWarnings("unchecked")
@Test
public void testCancelTokenSingleManager() throws Exception {
  for (int i = 0; i < TEST_RETRIES; i++) {
    DelegationTokenManager tm1 = null;
    String connectString = zkServer.getConnectString();
    Configuration conf = getSecretConf(connectString);
    tm1 = new DelegationTokenManager(conf, new Text("foo"));
    tm1.init();

    Token<DelegationTokenIdentifier> token =
        (Token<DelegationTokenIdentifier>)
        tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
    Assert.assertNotNull(token);
    tm1.cancelToken(token, "foo");
    try {
      verifyTokenFail(tm1, token);
      fail("Expected InvalidToken");
    } catch (SecretManager.InvalidToken it) {
      it.printStackTrace();
    }
    verifyDestroy(tm1, conf);
  }
}
 

@SuppressWarnings("unchecked")
@Test
public void testRenewTokenSingleManager() throws Exception {
  for (int i = 0; i < TEST_RETRIES; i++) {
    DelegationTokenManager tm1 = null;
    String connectString = zkServer.getConnectString();
    Configuration conf = getSecretConf(connectString);
    tm1 = new DelegationTokenManager(conf, new Text("foo"));
    tm1.init();

    Token<DelegationTokenIdentifier> token =
        (Token<DelegationTokenIdentifier>)
        tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
    Assert.assertNotNull(token);
    tm1.renewToken(token, "foo");
    tm1.verifyToken(token);
    verifyDestroy(tm1, conf);
  }
}
 

public MyResourceTracker4(Context context) {
  // create app Credentials
  org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token1 =
      new org.apache.hadoop.security.token.Token<DelegationTokenIdentifier>();
  token1.setKind(new Text("kind1"));
  expectedCredentials.addToken(new Text("token1"), token1);
  this.context = context;
}
 

/** Verifies that a configured delegation token binding is correctly loaded and employed */
@Test
public void testDelegationTokenBinding() throws IOException {
  URI initUri = new Path("gs://test/").toUri();
  Text expectedKind = TestTokenIdentifierImpl.KIND;

  GoogleHadoopFileSystem fs = new GoogleHadoopFileSystem();
  fs.initialize(initUri, loadConfig());

  // Request a delegation token
  Token<?> dt = fs.getDelegationToken(null);
  assertWithMessage("Expected a delegation token").that(dt).isNotNull();
  assertWithMessage("Unexpected delegation token service")
      .that(dt.getService().toString())
      .isEqualTo("gs://test");
  assertWithMessage("Unexpected delegation token kind")
      .that(dt.getKind())
      .isEqualTo(expectedKind);

  // Validate the associated identifier
  TokenIdentifier decoded = dt.decodeIdentifier();
  assertWithMessage("Failed to decode token identifier").that(decoded).isNotNull();
  assertWithMessage("Unexpected delegation token identifier type")
      .that(decoded)
      .isInstanceOf(TestTokenIdentifierImpl.class);

  DelegationTokenIdentifier identifier = (DelegationTokenIdentifier) decoded;
  assertWithMessage("Unexpected delegation token identifier kind")
      .that(identifier.getKind())
      .isEqualTo(expectedKind);
}
 

public static DelegationTokenIOException wrongTokenType(
    Class<? extends DelegationTokenIdentifier> expectedClass,
    DelegationTokenIdentifier identifier) {
  return new DelegationTokenIOException(
      String.format(
          "Delegation token type is incorrect;"
              + " expected a token identifier of type %s but got %s and kind %s",
          expectedClass, identifier.getClass(), identifier.getKind()));
}
 

private void verifyTokenFailWithRetry(DelegationTokenManager tm,
    Token<DelegationTokenIdentifier> token, int retryCount)
    throws IOException, InterruptedException {
  try {
    tm.verifyToken(token);
  } catch (SecretManager.InvalidToken er) {
    throw er;
  }
  if (retryCount > 0) {
    Thread.sleep(RETRY_WAIT);
    verifyTokenFailWithRetry(tm, token, retryCount - 1);
  }
}
 

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testStopThreads() throws Exception {
  DelegationTokenManager tm1 = null;
  String connectString = zkServer.getConnectString();

  // let's make the update interval short and the shutdown interval
  // comparatively longer, so if the update thread runs after shutdown,
  // it will cause an error.
  final long updateIntervalSeconds = 1;
  final long shutdownTimeoutMillis = updateIntervalSeconds * 1000 * 5;
  Configuration conf = getSecretConf(connectString);
  conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, updateIntervalSeconds);
  conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, updateIntervalSeconds);
  conf.setLong(DelegationTokenManager.RENEW_INTERVAL, updateIntervalSeconds);

  conf.setLong(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_SHUTDOWN_TIMEOUT, shutdownTimeoutMillis);
  tm1 = new DelegationTokenManager(conf, new Text("foo"));
  tm1.init();

  Token<DelegationTokenIdentifier> token =
    (Token<DelegationTokenIdentifier>)
  tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
  Assert.assertNotNull(token);

  AbstractDelegationTokenSecretManager sm = tm1.getDelegationTokenSecretManager();
  ZKDelegationTokenSecretManager zksm = (ZKDelegationTokenSecretManager)sm;
  ExecutorService es = zksm.getListenerThreadPool();
  es.submit(new Callable<Void>() {
    public Void call() throws Exception {
      Thread.sleep(shutdownTimeoutMillis * 2); // force this to be shutdownNow
      return null;
    }
  });

  tm1.destroy();
}
 

@Test
public void testNodeHeartBeatResponse() throws IOException {
  NodeHeartbeatResponse record =
      Records.newRecord(NodeHeartbeatResponse.class);
  Map<ApplicationId, ByteBuffer> appCredentials =
      new HashMap<ApplicationId, ByteBuffer>();
  Credentials app1Cred = new Credentials();

  Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>();
  token1.setKind(new Text("kind1"));
  app1Cred.addToken(new Text("token1"), token1);
  Token<DelegationTokenIdentifier> token2 =
      new Token<DelegationTokenIdentifier>();
  token2.setKind(new Text("kind2"));
  app1Cred.addToken(new Text("token2"), token2);

  DataOutputBuffer dob = new DataOutputBuffer();
  app1Cred.writeTokenStorageToStream(dob);
  ByteBuffer byteBuffer1 = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  appCredentials.put(ApplicationId.newInstance(1234, 1), byteBuffer1);
  record.setSystemCredentialsForApps(appCredentials);

  NodeHeartbeatResponse proto =
      new NodeHeartbeatResponsePBImpl(
        ((NodeHeartbeatResponsePBImpl) record).getProto());
  Assert.assertEquals(appCredentials, proto.getSystemCredentialsForApps());
}
 

private void verifyTokenFailWithRetry(DelegationTokenManager tm,
    Token<DelegationTokenIdentifier> token, int retryCount)
    throws IOException, InterruptedException {
  try {
    tm.verifyToken(token);
  } catch (SecretManager.InvalidToken er) {
    throw er;
  }
  if (retryCount > 0) {
    Thread.sleep(RETRY_WAIT);
    verifyTokenFailWithRetry(tm, token, retryCount - 1);
  }
}
 

/**
 * Get any bound DT or create a new one.
 *
 * @return a delegation token.
 * @throws IOException if one cannot be created
 */
@SuppressWarnings("OptionalGetWithoutIsPresent")
public Token<DelegationTokenIdentifier> getBoundOrNewDT(String renewer) throws IOException {
  logger.atFine().log("Delegation token requested");
  if (isBoundToDT()) {
    // the FS was created on startup with a token, so return it.
    logger.atFine().log("Returning current token");
    return getBoundDT();
  }

  // not bound to a token, so create a new one.
  // issued DTs are not cached so that long-lived filesystems can
  // reliably issue session/role tokens.
  return tokenBinding.createDelegationToken(renewer);
}
 

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testStopThreads() throws Exception {
  DelegationTokenManager tm1 = null;
  String connectString = zkServer.getConnectString();

  // let's make the update interval short and the shutdown interval
  // comparatively longer, so if the update thread runs after shutdown,
  // it will cause an error.
  final long updateIntervalSeconds = 1;
  final long shutdownTimeoutMillis = updateIntervalSeconds * 1000 * 5;
  Configuration conf = getSecretConf(connectString);
  conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, updateIntervalSeconds);
  conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, updateIntervalSeconds);
  conf.setLong(DelegationTokenManager.RENEW_INTERVAL, updateIntervalSeconds);

  conf.setLong(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_SHUTDOWN_TIMEOUT, shutdownTimeoutMillis);
  tm1 = new DelegationTokenManager(conf, new Text("foo"));
  tm1.init();

  Token<DelegationTokenIdentifier> token =
    (Token<DelegationTokenIdentifier>)
  tm1.createToken(UserGroupInformation.getCurrentUser(), "foo");
  Assert.assertNotNull(token);

  AbstractDelegationTokenSecretManager sm = tm1.getDelegationTokenSecretManager();
  ZKDelegationTokenSecretManager zksm = (ZKDelegationTokenSecretManager)sm;
  ExecutorService es = zksm.getListenerThreadPool();
  es.submit(new Callable<Void>() {
    public Void call() throws Exception {
      Thread.sleep(shutdownTimeoutMillis * 2); // force this to be shutdownNow
      return null;
    }
  });

  tm1.destroy();
}
 

public MyResourceTracker4(Context context) {
  // create app Credentials
  org.apache.hadoop.security.token.Token<DelegationTokenIdentifier> token1 =
      new org.apache.hadoop.security.token.Token<DelegationTokenIdentifier>();
  token1.setKind(new Text("kind1"));
  expectedCredentials.addToken(new Text("token1"), token1);
  this.context = context;
}
 

@Test
public void testNodeHeartBeatResponse() throws IOException {
  NodeHeartbeatResponse record =
      Records.newRecord(NodeHeartbeatResponse.class);
  Map<ApplicationId, ByteBuffer> appCredentials =
      new HashMap<ApplicationId, ByteBuffer>();
  Credentials app1Cred = new Credentials();

  Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>();
  token1.setKind(new Text("kind1"));
  app1Cred.addToken(new Text("token1"), token1);
  Token<DelegationTokenIdentifier> token2 =
      new Token<DelegationTokenIdentifier>();
  token2.setKind(new Text("kind2"));
  app1Cred.addToken(new Text("token2"), token2);

  DataOutputBuffer dob = new DataOutputBuffer();
  app1Cred.writeTokenStorageToStream(dob);
  ByteBuffer byteBuffer1 = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  appCredentials.put(ApplicationId.newInstance(1234, 1), byteBuffer1);
  record.setSystemCredentialsForApps(appCredentials);

  NodeHeartbeatResponse proto =
      new NodeHeartbeatResponsePBImpl(
        ((NodeHeartbeatResponsePBImpl) record).getProto());
  Assert.assertEquals(appCredentials, proto.getSystemCredentialsForApps());
}
 

/**
 * Verify that a token identifier is of a specific class. This will reject subclasses (i.e. it is
 * stricter than {@code instanceof}, then cast it to that type.
 *
 * @param identifier identifier to validate
 * @param expectedClass class of the expected token identifier.
 * @throws DelegationTokenIOException If the wrong class was found.
 */
@SuppressWarnings("unchecked") // safe by contract of convertTokenIdentifier()
protected <T extends DelegationTokenIdentifier> T convertTokenIdentifier(
    DelegationTokenIdentifier identifier, Class<T> expectedClass)
    throws DelegationTokenIOException {
  if (identifier.getClass().equals(expectedClass)) {
    return (T) identifier;
  }
  throw DelegationTokenIOException.wrongTokenType(expectedClass, identifier);
}
 

@Override
public DelegationTokenIdentifier createIdentifier() {
  return AbstractDelegationTokenBinding.this.createEmptyIdentifier();
}
 

@Override
public DelegationTokenIdentifier createEmptyIdentifier() {
  return new TestTokenIdentifierImpl();
}
 

@Override
public AccessTokenProvider bindToTokenIdentifier(DelegationTokenIdentifier retrievedIdentifier)
    throws IOException {
  return deployUnbonded();
}
 

@Override
public DelegationTokenIdentifier createTokenIdentifier() throws IOException {
  return createEmptyIdentifier();
}
 

@Override
public AccessTokenProvider bindToTokenIdentifier(DelegationTokenIdentifier retrievedIdentifier) throws IOException {
    return new BrokerAccessTokenProvider(getService(), (BrokerTokenIdentifier) retrievedIdentifier);
}
 

@Override
public byte[] retrievePassword(DelegationTokenIdentifier identifier) throws InvalidToken {
  return pwd;
}
 

@Override
protected byte[] createPassword(DelegationTokenIdentifier identifier) {
  return pwd;
}
 

private void verifyTokenFail(DelegationTokenManager tm,
    Token<DelegationTokenIdentifier> token) throws IOException,
    InterruptedException {
  verifyTokenFailWithRetry(tm, token, RETRY_COUNT);
}
 

private void verifyTokenFail(DelegationTokenManager tm,
    Token<DelegationTokenIdentifier> token) throws IOException,
    InterruptedException {
  verifyTokenFailWithRetry(tm, token, RETRY_COUNT);
}
 

@Override
public DelegationTokenIdentifier createEmptyIdentifier() {
    return new BrokerTokenIdentifier();
}