org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier Java Examples

private WebHdfsFileSystem getWebHdfsFileSystem(UserGroupInformation ugi,
    Configuration conf) throws IOException {
  if (UserGroupInformation.isSecurityEnabled()) {
    DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
        ugi.getUserName()), null, null);
    FSNamesystem namesystem = mock(FSNamesystem.class);
    DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
        86400000, 86400000, 86400000, 86400000, namesystem);
    dtSecretManager.startThreads();
    Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
        dtId, dtSecretManager);
    SecurityUtil.setTokenService(
        token, NetUtils.createSocketAddr(uri.getAuthority()));
    token.setKind(WebHdfsFileSystem.TOKEN_KIND);
    ugi.addToken(token);
  }
  return (WebHdfsFileSystem) FileSystem.get(uri, conf);
}
 

@Test 
public void testCancelDelegationToken() throws Exception {
  Token<DelegationTokenIdentifier> token = generateDelegationToken(
      "SomeUser", "JobTracker");
  //Fake renewer should not be able to renew
  try {
    dtSecretManager.cancelToken(token, "FakeCanceller");
    Assert.fail("should have failed");
  } catch (AccessControlException ace) {
    // PASS
  }
  dtSecretManager.cancelToken(token, "JobTracker");
  try {
    dtSecretManager.renewToken(token, "JobTracker");
    Assert.fail("should have failed");
  } catch (InvalidToken it) {
    // PASS
  }
}
 

@Override
public void handle(Channel channel, Token<DelegationTokenIdentifier> token,
    String serviceUrl) throws IOException {
  Assert.assertEquals(testToken, token);

  Credentials creds = new Credentials();
  creds.addToken(new Text(serviceUrl), token);
  DataOutputBuffer out = new DataOutputBuffer();
  creds.write(out);
  int fileLength = out.getData().length;
  ChannelBuffer cbuffer = ChannelBuffers.buffer(fileLength);
  cbuffer.writeBytes(out.getData());
  HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  response.setHeader(HttpHeaders.Names.CONTENT_LENGTH,
      String.valueOf(fileLength));
  response.setContent(cbuffer);
  channel.write(response).addListener(ChannelFutureListener.CLOSE);
}
 

private static void doRenewOrCancel(
    final Token<DelegationTokenIdentifier> token, final Configuration conf,
    final TokenTestAction action)
    throws IOException, InterruptedException {
  UserGroupInformation.createRemoteUser("JobTracker").doAs(
      new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          switch (action) {
          case RENEW:
            token.renew(conf);
            break;
          case CANCEL:
            token.cancel(conf);
            break;
          default:
            fail("bad action:" + action);
          }
          return null;
        }
      });
}
 

@Override
public Token<DelegationTokenIdentifier> getDelegationToken(
    final String renewer) throws IOException {
  final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
  Token<DelegationTokenIdentifier> token =
      new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
          op, null, new RenewerParam(renewer)) {
    @Override
    Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
        throws IOException {
      return JsonUtil.toDelegationToken(json);
    }
  }.run();
  if (token != null) {
    token.setService(tokenServiceName);
  } else {
    if (disallowFallbackToInsecureCluster) {
      throw new AccessControlException(CANT_FALLBACK_TO_INSECURE_MSG);
    }
  }
  return token;
}
 

@Override
public Token<DelegationTokenIdentifier> getDelegationToken(
    final String renewer) throws IOException {
  final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
  Token<DelegationTokenIdentifier> token =
      new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
          op, null, new RenewerParam(renewer)) {
    @Override
    Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
        throws IOException {
      return JsonUtil.toDelegationToken(json);
    }
  }.run();
  if (token != null) {
    token.setService(tokenServiceName);
  } else {
    if (disallowFallbackToInsecureCluster) {
      throw new AccessControlException(CANT_FALLBACK_TO_INSECURE_MSG);
    }
  }
  return token;
}
 

@Override
public synchronized <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
  /**
   * XXX The kind of the token has been changed by DelegationTokenFetcher. We
   * use the token for renewal, since the reflection utilities needs the value
   * of the kind field to correctly renew the token.
   *
   * For other operations, however, the client has to send a
   * HDFS_DELEGATION_KIND token over the wire so that it can talk to Hadoop
   * 0.20.203 clusters. Later releases fix this problem. See HDFS-5440 for
   * more details.
   */
  renewToken = token;
  delegationToken = new Token<T>(token);
  delegationToken.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
}
 

private WebHdfsFileSystem getWebHdfsFileSystem(UserGroupInformation ugi,
    Configuration conf) throws IOException {
  if (UserGroupInformation.isSecurityEnabled()) {
    DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
        ugi.getUserName()), null, null);
    FSNamesystem namesystem = mock(FSNamesystem.class);
    DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
        86400000, 86400000, 86400000, 86400000, namesystem);
    dtSecretManager.startThreads();
    Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
        dtId, dtSecretManager);
    SecurityUtil.setTokenService(
        token, NetUtils.createSocketAddr(uri.getAuthority()));
    token.setKind(WebHdfsFileSystem.TOKEN_KIND);
    ugi.addToken(token);
  }
  return (WebHdfsFileSystem) FileSystem.get(uri, conf);
}
 

/**
 * @see ClientProtocol#getDelegationToken(Text)
 */
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
    throws IOException {
  assert dtService != null;
  TraceScope scope = Trace.startSpan("getDelegationToken", traceSampler);
  try {
    Token<DelegationTokenIdentifier> token =
      namenode.getDelegationToken(renewer);
    if (token != null) {
      token.setService(this.dtService);
      LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
    } else {
      LOG.info("Cannot get delegation token from " + renewer);
    }
    return token;
  } finally {
    scope.close();
  }
}
 

private static void doRenewOrCancel(
    final Token<DelegationTokenIdentifier> token, final Configuration conf,
    final TokenTestAction action)
    throws IOException, InterruptedException {
  UserGroupInformation.createRemoteUser("JobTracker").doAs(
      new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          switch (action) {
          case RENEW:
            token.renew(conf);
            break;
          case CANCEL:
            token.cancel(conf);
            break;
          default:
            fail("bad action:" + action);
          }
          return null;
        }
      });
}
 

@SuppressWarnings("unchecked")
@Override
public long renewDelegationToken(final Token<?> token) throws IOException {
  // update the kerberos credentials, if they are coming from a keytab
  UserGroupInformation connectUgi = ugi.getRealUser();
  if (connectUgi == null) {
    connectUgi = ugi;
  }
  try {
    return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
      @Override
      public Long run() throws Exception {
        InetSocketAddress serviceAddr = SecurityUtil
            .getTokenServiceAddr(token);
        return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
            DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
            (Token<DelegationTokenIdentifier>) token);
      }
    });
  } catch (InterruptedException e) {
    throw new IOException(e);
  }
}
 

private static ClientProtocol getNNProxy(
    Token<DelegationTokenIdentifier> token, Configuration conf)
    throws IOException {
  URI uri = HAUtil.getServiceUriFromToken(HdfsConstants.HDFS_URI_SCHEME,
          token);
  if (HAUtil.isTokenForLogicalUri(token) &&
      !HAUtil.isLogicalUri(conf, uri)) {
    // If the token is for a logical nameservice, but the configuration
    // we have disagrees about that, we can't actually renew it.
    // This can be the case in MR, for example, if the RM doesn't
    // have all of the HA clusters configured in its configuration.
    throw new IOException("Unable to map logical nameservice URI '" +
        uri + "' to a NameNode. Local configuration does not have " +
        "a failover proxy provider configured.");
  }
  
  NameNodeProxies.ProxyAndInfo<ClientProtocol> info =
    NameNodeProxies.createProxy(conf, uri, ClientProtocol.class);
  assert info.getDelegationTokenService().equals(token.getService()) :
    "Returned service '" + info.getDelegationTokenService().toString() +
    "' doesn't match expected service '" +
    token.getService().toString() + "'";
    
  return info.getProxy();
}
 

private static ClientProtocol getNNProxy(
    Token<DelegationTokenIdentifier> token, Configuration conf)
    throws IOException {
  URI uri = HAUtil.getServiceUriFromToken(HdfsConstants.HDFS_URI_SCHEME,
          token);
  if (HAUtil.isTokenForLogicalUri(token) &&
      !HAUtil.isLogicalUri(conf, uri)) {
    // If the token is for a logical nameservice, but the configuration
    // we have disagrees about that, we can't actually renew it.
    // This can be the case in MR, for example, if the RM doesn't
    // have all of the HA clusters configured in its configuration.
    throw new IOException("Unable to map logical nameservice URI '" +
        uri + "' to a NameNode. Local configuration does not have " +
        "a failover proxy provider configured.");
  }
  
  NameNodeProxies.ProxyAndInfo<ClientProtocol> info =
    NameNodeProxies.createProxy(conf, uri, ClientProtocol.class);
  assert info.getDelegationTokenService().equals(token.getService()) :
    "Returned service '" + info.getDelegationTokenService().toString() +
    "' doesn't match expected service '" +
    token.getService().toString() + "'";
    
  return info.getProxy();
}
 

/**
 * Auxiliary - create token
 * @param renewer
 * @return
 * @throws IOException
 */
static MyToken createTokens(Text renewer) 
  throws IOException {
  Text user1= new Text("user1");
  
  MyDelegationTokenSecretManager sm = new MyDelegationTokenSecretManager(
      DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
      DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
      DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
      3600000, null);
  sm.startThreads();
  
  DelegationTokenIdentifier dtId1 = 
    new DelegationTokenIdentifier(user1, renewer, user1);
  
  MyToken token1 = new MyToken(dtId1, sm);
 
  token1.setService(new Text("localhost:0"));
  return token1;
}
 

public static void delegationTokenToXml(ContentHandler contentHandler,
    DelegationTokenIdentifier token) throws SAXException {
  contentHandler.startElement("", "", "DELEGATION_TOKEN_IDENTIFIER", new AttributesImpl());
  XMLUtils.addSaxString(contentHandler, "KIND", token.getKind().toString());
  XMLUtils.addSaxString(contentHandler, "SEQUENCE_NUMBER",
      Integer.toString(token.getSequenceNumber()));
  XMLUtils.addSaxString(contentHandler, "OWNER",
      token.getOwner().toString());
  XMLUtils.addSaxString(contentHandler, "RENEWER",
      token.getRenewer().toString());
  XMLUtils.addSaxString(contentHandler, "REALUSER",
      token.getRealUser().toString());
  XMLUtils.addSaxString(contentHandler, "ISSUE_DATE",
      Long.toString(token.getIssueDate()));
  XMLUtils.addSaxString(contentHandler, "MAX_DATE",
      Long.toString(token.getMaxDate()));
  XMLUtils.addSaxString(contentHandler, "MASTER_KEY_ID",
      Integer.toString(token.getMasterKeyId()));
  contentHandler.endElement("", "", "DELEGATION_TOKEN_IDENTIFIER");
}
 

private ServerBootstrap startHttpServer(int port,
    final Token<DelegationTokenIdentifier> token, final URI url) {
  ServerBootstrap bootstrap = new ServerBootstrap(
      new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
          Executors.newCachedThreadPool()));

  bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
    @Override
    public ChannelPipeline getPipeline() throws Exception {
      return Channels.pipeline(new HttpRequestDecoder(),
          new HttpChunkAggregator(65536), new HttpResponseEncoder(),
          new CredentialsLogicHandler(token, url.toString()));
    }
  });
  bootstrap.bind(new InetSocketAddress("localhost", port));
  return bootstrap;
}
 

@Test(timeout=20000)
public void testDelegationTokenWithRealUser() throws IOException {
  try {
    Token<?>[] tokens = proxyUgi
        .doAs(new PrivilegedExceptionAction<Token<?>[]>() {
          @Override
          public Token<?>[] run() throws IOException {
            return cluster.getFileSystem().addDelegationTokens("RenewerUser", null);
          }
        });
    DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
    byte[] tokenId = tokens[0].getIdentifier();
    identifier.readFields(new DataInputStream(new ByteArrayInputStream(
        tokenId)));
    Assert.assertEquals(identifier.getUser().getUserName(), PROXY_USER);
    Assert.assertEquals(identifier.getUser().getRealUser().getUserName(),
        REAL_USER);
  } catch (InterruptedException e) {
    //Do Nothing
  }
}
 

Token<DelegationTokenIdentifier> delegationToken() throws IOException {
  String delegation = param(DelegationParam.NAME);
  final Token<DelegationTokenIdentifier> token = new
    Token<DelegationTokenIdentifier>();
  token.decodeFromUrlString(delegation);
  URI nnUri = URI.create(HDFS_URI_SCHEME + "://" + namenodeId());
  boolean isLogical = HAUtil.isLogicalUri(conf, nnUri);
  if (isLogical) {
    token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri,
      HDFS_URI_SCHEME));
  } else {
    token.setService(SecurityUtil.buildTokenService(nnUri));
  }
  return token;
}
 

/**
 * log delegation token to edit log
 * @param id DelegationTokenIdentifier
 * @param expiryTime of the token
 */
void logGetDelegationToken(DelegationTokenIdentifier id,
    long expiryTime) {
  GetDelegationTokenOp op = GetDelegationTokenOp.getInstance(cache.get())
    .setDelegationTokenIdentifier(id)
    .setExpiryTime(expiryTime);
  logEdit(op);
}
 

@Override
public void handle(Channel channel, Token<DelegationTokenIdentifier> token,
    String serviceUrl) throws IOException {
  Assert.assertEquals(testToken, token);
  byte[] bytes = EXP_DATE.getBytes();
  ChannelBuffer cbuffer = ChannelBuffers.buffer(bytes.length);
  cbuffer.writeBytes(bytes);
  HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  response.setHeader(HttpHeaders.Names.CONTENT_LENGTH,
      String.valueOf(bytes.length));
  response.setContent(cbuffer);
  channel.write(response).addListener(ChannelFutureListener.CLOSE);
}
 

/**
 * Locate a delegation token associated with the given HA cluster URI, and if
 * one is found, clone it to also represent the underlying namenode address.
 * @param ugi the UGI to modify
 * @param haUri the logical URI for the cluster
 * @param nnAddrs collection of NNs in the cluster to which the token
 * applies
 */
public static void cloneDelegationTokenForLogicalUri(
    UserGroupInformation ugi, URI haUri,
    Collection<InetSocketAddress> nnAddrs) {
  // this cloning logic is only used by hdfs
  Text haService = HAUtil.buildTokenServiceForLogicalUri(haUri,
      HdfsConstants.HDFS_URI_SCHEME);
  Token<DelegationTokenIdentifier> haToken =
      tokenSelector.selectToken(haService, ugi.getTokens());
  if (haToken != null) {
    for (InetSocketAddress singleNNAddr : nnAddrs) {
      // this is a minor hack to prevent physical HA tokens from being
      // exposed to the user via UGI.getCredentials(), otherwise these
      // cloned tokens may be inadvertently propagated to jobs
      Token<DelegationTokenIdentifier> specificToken =
          new Token.PrivateToken<DelegationTokenIdentifier>(haToken);
      SecurityUtil.setTokenService(specificToken, singleNNAddr);
      Text alias = new Text(
          buildTokenServicePrefixForLogicalUri(HdfsConstants.HDFS_URI_SCHEME)
              + "//" + specificToken.getService());
      ugi.addToken(alias, specificToken);
      LOG.debug("Mapped HA service delegation token for logical URI " +
          haUri + " to namenode " + singleNNAddr);
    }
  } else {
    LOG.debug("No HA service delegation token found for logical URI " +
        haUri);
  }
}
 

private void injectToken() throws IOException {
  if (UserGroupInformation.isSecurityEnabled()) {
    Token<DelegationTokenIdentifier> token = params.delegationToken();
    token.setKind(HDFS_DELEGATION_KIND);
    ugi.addToken(token);
  }
}
 

@Override
public void handle(Channel channel, Token<DelegationTokenIdentifier> token,
    String serviceUrl) throws IOException {
  Assert.assertEquals(testToken, token);
  HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  channel.write(response).addListener(ChannelFutureListener.CLOSE);
}
 

@Override
public void handle(Channel channel, Token<DelegationTokenIdentifier> token,
    String serviceUrl) throws IOException {
  Assert.assertEquals(testToken, token);
  byte[] bytes = EXP_DATE.getBytes();
  ChannelBuffer cbuffer = ChannelBuffers.buffer(bytes.length);
  cbuffer.writeBytes(bytes);
  HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  response.setHeader(HttpHeaders.Names.CONTENT_LENGTH,
      String.valueOf(bytes.length));
  response.setContent(cbuffer);
  channel.write(response).addListener(ChannelFutureListener.CLOSE);
}
 

/** Convert a Json map to a Token of DelegationTokenIdentifier. */
@SuppressWarnings("unchecked")
public static Token<DelegationTokenIdentifier> toDelegationToken(
    final Map<?, ?> json) throws IOException {
  final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
  return (Token<DelegationTokenIdentifier>)toToken(m);
}
 

/**
 * Renew a delegation token
 * @param token the token to renew
 * @return the new expiration time
 * @throws InvalidToken
 * @throws IOException
 * @deprecated Use Token.renew instead.
 */
@Deprecated
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
    throws InvalidToken, IOException {
  LOG.info("Renewing " + DelegationTokenIdentifier.stringifyToken(token));
  try {
    return token.renew(conf);
  } catch (InterruptedException ie) {                                       
    throw new RuntimeException("caught interrupted", ie);
  } catch (RemoteException re) {
    throw re.unwrapRemoteException(InvalidToken.class,
                                   AccessControlException.class);
  }
}
 

/**
 * HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an
 * exception if the URI is a logical URI. This bug fails the combination of
 * ha + mapred + security.
 */
@Test(timeout = 300000)
public void testDFSGetCanonicalServiceName() throws Exception {
  URI hAUri = HATestUtil.getLogicalUri(cluster);
  String haService = HAUtil.buildTokenServiceForLogicalUri(hAUri,
      HdfsConstants.HDFS_URI_SCHEME).toString();
  assertEquals(haService, dfs.getCanonicalServiceName());
  final String renewer = UserGroupInformation.getCurrentUser().getShortUserName();
  final Token<DelegationTokenIdentifier> token =
      getDelegationToken(dfs, renewer);
  assertEquals(haService, token.getService().toString());
  // make sure the logical uri is handled correctly
  token.renew(dfs.getConf());
  token.cancel(dfs.getConf());
}
 

@Test
public void testDeserializeHAToken() throws IOException {
  Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
  final Token<DelegationTokenIdentifier> token = new
      Token<DelegationTokenIdentifier>();
  QueryStringDecoder decoder = new QueryStringDecoder(
    WebHdfsHandler.WEBHDFS_PREFIX + "/?"
    + NamenodeAddressParam.NAME + "=" + LOGICAL_NAME + "&"
    + DelegationParam.NAME + "=" + token.encodeToUrlString());
  ParameterParser testParser = new ParameterParser(decoder, conf);
  final Token<DelegationTokenIdentifier> tok2 = testParser.delegationToken();
  Assert.assertTrue(HAUtil.isTokenForLogicalUri(tok2));
}
 

@Override
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
    throws IOException {
  CancelDelegationTokenRequestProto req = CancelDelegationTokenRequestProto
      .newBuilder()
      .setToken(PBHelper.convert(token))
      .build();
  try {
    rpcProxy.cancelDelegationToken(null, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

@Test
public void testDelegationTokenSecretManager() throws Exception {
  Token<DelegationTokenIdentifier> token = generateDelegationToken(
      "SomeUser", "JobTracker");
  // Fake renewer should not be able to renew
  try {
	  dtSecretManager.renewToken(token, "FakeRenewer");
	  Assert.fail("should have failed");
  } catch (AccessControlException ace) {
    // PASS
  }
 dtSecretManager.renewToken(token, "JobTracker");
  DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
  byte[] tokenId = token.getIdentifier();
  identifier.readFields(new DataInputStream(
           new ByteArrayInputStream(tokenId)));
  Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
  LOG.info("Sleep to expire the token");
 Thread.sleep(6000);
 //Token should be expired
 try {
   dtSecretManager.retrievePassword(identifier);
   //Should not come here
   Assert.fail("Token should have expired");
 } catch (InvalidToken e) {
   //Success
 }
 dtSecretManager.renewToken(token, "JobTracker");
 LOG.info("Sleep beyond the max lifetime");
 Thread.sleep(5000);
 try {
	  dtSecretManager.renewToken(token, "JobTracker");
	  Assert.fail("should have been expired");
 } catch (InvalidToken it) {
   // PASS
 }
}