Java Code Examples for org.apache.hadoop.hdfs.DFSUtil#getHttpPolicy()

The following examples show how to use org.apache.hadoop.hdfs.DFSUtil#getHttpPolicy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Nfs3HttpServer.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
void start() throws IOException {
  final InetSocketAddress httpAddr = getHttpAddress(conf);

  final String httpsAddrString = conf.get(
      NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY,
      NfsConfigKeys.NFS_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "nfs3",
      NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY,
      NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY);

  this.httpServer = builder.build();
  this.httpServer.start();
  
  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    infoPort = httpServer.getConnectorAddress(connIdx++).getPort();
  }

  if (policy.isHttpsEnabled()) {
    infoSecurePort = httpServer.getConnectorAddress(connIdx).getPort();
  }
}
Example 2
Source File: DataNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks if the DataNode has a secure configuration if security is enabled.
 * There are 2 possible configurations that are considered secure:
 * 1. The server has bound to privileged ports for RPC and HTTP via
 *   SecureDataNodeStarter.
 * 2. The configuration enables SASL on DataTransferProtocol and HTTPS (no
 *   plain HTTP) for the HTTP server.  The SASL handshake guarantees
 *   authentication of the RPC server before a client transmits a secret, such
 *   as a block access token.  Similarly, SSL guarantees authentication of the
 *   HTTP server before a client transmits a secret, such as a delegation
 *   token.
 * It is not possible to run with both privileged ports and SASL on
 * DataTransferProtocol.  For backwards-compatibility, the connection logic
 * must check if the target port is a privileged port, and if so, skip the
 * SASL handshake.
 *
 * @param dnConf DNConf to check
 * @param conf Configuration to check
 * @param resources SecuredResources obtained for DataNode
 * @throws RuntimeException if security enabled, but configuration is insecure
 */
private static void checkSecureConfig(DNConf dnConf, Configuration conf,
    SecureResources resources) throws RuntimeException {
  if (!UserGroupInformation.isSecurityEnabled()) {
    return;
  }
  SaslPropertiesResolver saslPropsResolver = dnConf.getSaslPropsResolver();
  if (resources != null && saslPropsResolver == null) {
    return;
  }
  if (dnConf.getIgnoreSecurePortsForTesting()) {
    return;
  }
  if (saslPropsResolver != null &&
      DFSUtil.getHttpPolicy(conf) == HttpConfig.Policy.HTTPS_ONLY &&
      resources == null) {
    return;
  }
  throw new RuntimeException("Cannot start secure DataNode without " +
    "configuring either privileged resources or SASL RPC data transfer " +
    "protection and SSL for HTTP.  Using privileged resources in " +
    "combination with SASL RPC data transfer protection is not supported.");
}
Example 3
Source File: Nfs3HttpServer.java    From big-c with Apache License 2.0 6 votes vote down vote up 
void start() throws IOException {
  final InetSocketAddress httpAddr = getHttpAddress(conf);

  final String httpsAddrString = conf.get(
      NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY,
      NfsConfigKeys.NFS_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "nfs3",
      NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY,
      NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY);

  this.httpServer = builder.build();
  this.httpServer.start();
  
  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    infoPort = httpServer.getConnectorAddress(connIdx++).getPort();
  }

  if (policy.isHttpsEnabled()) {
    infoSecurePort = httpServer.getConnectorAddress(connIdx).getPort();
  }
}
Example 4
Source File: DataNode.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks if the DataNode has a secure configuration if security is enabled.
 * There are 2 possible configurations that are considered secure:
 * 1. The server has bound to privileged ports for RPC and HTTP via
 *   SecureDataNodeStarter.
 * 2. The configuration enables SASL on DataTransferProtocol and HTTPS (no
 *   plain HTTP) for the HTTP server.  The SASL handshake guarantees
 *   authentication of the RPC server before a client transmits a secret, such
 *   as a block access token.  Similarly, SSL guarantees authentication of the
 *   HTTP server before a client transmits a secret, such as a delegation
 *   token.
 * It is not possible to run with both privileged ports and SASL on
 * DataTransferProtocol.  For backwards-compatibility, the connection logic
 * must check if the target port is a privileged port, and if so, skip the
 * SASL handshake.
 *
 * @param dnConf DNConf to check
 * @param conf Configuration to check
 * @param resources SecuredResources obtained for DataNode
 * @throws RuntimeException if security enabled, but configuration is insecure
 */
private static void checkSecureConfig(DNConf dnConf, Configuration conf,
    SecureResources resources) throws RuntimeException {
  if (!UserGroupInformation.isSecurityEnabled()) {
    return;
  }
  SaslPropertiesResolver saslPropsResolver = dnConf.getSaslPropsResolver();
  if (resources != null && saslPropsResolver == null) {
    return;
  }
  if (dnConf.getIgnoreSecurePortsForTesting()) {
    return;
  }
  if (saslPropsResolver != null &&
      DFSUtil.getHttpPolicy(conf) == HttpConfig.Policy.HTTPS_ONLY &&
      resources == null) {
    return;
  }
  throw new RuntimeException("Cannot start secure DataNode without " +
    "configuring either privileged resources or SASL RPC data transfer " +
    "protection and SSL for HTTP.  Using privileged resources in " +
    "combination with SASL RPC data transfer protection is not supported.");
}
Example 5
Source File: SecondaryNameNode.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Initialize SecondaryNameNode.
 */
private void initialize(final Configuration conf,
    CommandLineOpts commandLineOpts) throws IOException {
  final InetSocketAddress infoSocAddr = getHttpAddress(conf);
  final String infoBindAddress = infoSocAddr.getHostName();
  UserGroupInformation.setConfiguration(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    SecurityUtil.login(conf,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY, infoBindAddress);
  }
  // initiate Java VM metrics
  DefaultMetricsSystem.initialize("SecondaryNameNode");
  JvmMetrics.create("SecondaryNameNode",
      conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
      DefaultMetricsSystem.instance());

  // Create connection to the namenode.
  shouldRun = true;
  nameNodeAddr = NameNode.getServiceAddress(conf, true);

  this.conf = conf;
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nameNodeAddr, 
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();

  // initialize checkpoint directories
  fsName = getInfoServer();
  checkpointDirs = FSImage.getCheckpointDirs(conf,
                                "/tmp/hadoop/dfs/namesecondary");
  checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, 
                                "/tmp/hadoop/dfs/namesecondary");    
  checkpointImage = new CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs);
  checkpointImage.recoverCreate(commandLineOpts.shouldFormat());
  checkpointImage.deleteTempEdits();
  
  namesystem = new FSNamesystem(conf, checkpointImage, true);

  // Disable quota checks
  namesystem.dir.disableQuotaChecks();

  // Initialize other scheduling parameters from the configuration
  checkpointConf = new CheckpointConf(conf);

  final InetSocketAddress httpAddr = infoSocAddr;

  final String httpsAddrString = conf.getTrimmed(
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "secondary",
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY);

  nameNodeStatusBeanName = MBeans.register("SecondaryNameNode",
          "SecondaryNameNodeInfo", this);

  infoServer = builder.build();

  infoServer.setAttribute("secondary.name.node", this);
  infoServer.setAttribute("name.system.image", checkpointImage);
  infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
  infoServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
      ImageServlet.class, true);
  infoServer.start();

  LOG.info("Web server init done");

  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    InetSocketAddress httpAddress = infoServer.getConnectorAddress(connIdx++);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
        NetUtils.getHostPortString(httpAddress));
  }

  if (policy.isHttpsEnabled()) {
    InetSocketAddress httpsAddress = infoServer.getConnectorAddress(connIdx);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
        NetUtils.getHostPortString(httpsAddress));
  }

  legacyOivImageDir = conf.get(
      DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY);

  LOG.info("Checkpoint Period   :" + checkpointConf.getPeriod() + " secs "
      + "(" + checkpointConf.getPeriod() / 60 + " min)");
  LOG.info("Log Size Trigger    :" + checkpointConf.getTxnCount() + " txns");
}
Example 6
Source File: SecureDataNodeStarter.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Acquire privileged resources (i.e., the privileged ports) for the data
 * node. The privileged resources consist of the port of the RPC server and
 * the port of HTTP (not HTTPS) server.
 */
@VisibleForTesting
public static SecureResources getSecureResources(Configuration conf)
    throws Exception {
  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  boolean isSecure = UserGroupInformation.isSecurityEnabled();

  // Obtain secure port for data streaming to datanode
  InetSocketAddress streamingAddr  = DataNode.getStreamingAddr(conf);
  int socketWriteTimeout = conf.getInt(
      DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
      HdfsServerConstants.WRITE_TIMEOUT);

  ServerSocket ss = (socketWriteTimeout > 0) ? 
      ServerSocketChannel.open().socket() : new ServerSocket();
  ss.bind(streamingAddr, 0);

  // Check that we got the port we need
  if (ss.getLocalPort() != streamingAddr.getPort()) {
    throw new RuntimeException(
        "Unable to bind on specified streaming port in secure "
            + "context. Needed " + streamingAddr.getPort() + ", got "
            + ss.getLocalPort());
  }

  if (!SecurityUtil.isPrivilegedPort(ss.getLocalPort()) && isSecure) {
    throw new RuntimeException(
      "Cannot start secure datanode with unprivileged RPC ports");
  }

  System.err.println("Opened streaming server at " + streamingAddr);

  // Bind a port for the web server. The code intends to bind HTTP server to
  // privileged port only, as the client can authenticate the server using
  // certificates if they are communicating through SSL.
  final ServerSocketChannel httpChannel;
  if (policy.isHttpEnabled()) {
    httpChannel = ServerSocketChannel.open();
    InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf);
    httpChannel.socket().bind(infoSocAddr);
    InetSocketAddress localAddr = (InetSocketAddress) httpChannel.socket()
      .getLocalSocketAddress();

    if (localAddr.getPort() != infoSocAddr.getPort()) {
      throw new RuntimeException("Unable to bind on specified info port in secure " +
          "context. Needed " + streamingAddr.getPort() + ", got " + ss.getLocalPort());
    }
    System.err.println("Successfully obtained privileged resources (streaming port = "
        + ss + " ) (http listener port = " + localAddr.getPort() +")");

    if (localAddr.getPort() > 1023 && isSecure) {
      throw new RuntimeException(
          "Cannot start secure datanode with unprivileged HTTP ports");
    }
    System.err.println("Opened info server at " + infoSocAddr);
  } else {
    httpChannel = null;
  }

  return new SecureResources(ss, httpChannel);
}
Example 7
Source File: SecondaryNameNode.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Initialize SecondaryNameNode.
 */
private void initialize(final Configuration conf,
    CommandLineOpts commandLineOpts) throws IOException {
  final InetSocketAddress infoSocAddr = getHttpAddress(conf);
  final String infoBindAddress = infoSocAddr.getHostName();
  UserGroupInformation.setConfiguration(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    SecurityUtil.login(conf,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY, infoBindAddress);
  }
  // initiate Java VM metrics
  DefaultMetricsSystem.initialize("SecondaryNameNode");
  JvmMetrics.create("SecondaryNameNode",
      conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
      DefaultMetricsSystem.instance());

  // Create connection to the namenode.
  shouldRun = true;
  nameNodeAddr = NameNode.getServiceAddress(conf, true);

  this.conf = conf;
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nameNodeAddr, 
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();

  // initialize checkpoint directories
  fsName = getInfoServer();
  checkpointDirs = FSImage.getCheckpointDirs(conf,
                                "/tmp/hadoop/dfs/namesecondary");
  checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, 
                                "/tmp/hadoop/dfs/namesecondary");    
  checkpointImage = new CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs);
  checkpointImage.recoverCreate(commandLineOpts.shouldFormat());
  checkpointImage.deleteTempEdits();
  
  namesystem = new FSNamesystem(conf, checkpointImage, true);

  // Disable quota checks
  namesystem.dir.disableQuotaChecks();

  // Initialize other scheduling parameters from the configuration
  checkpointConf = new CheckpointConf(conf);

  final InetSocketAddress httpAddr = infoSocAddr;

  final String httpsAddrString = conf.getTrimmed(
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "secondary",
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY);

  nameNodeStatusBeanName = MBeans.register("SecondaryNameNode",
          "SecondaryNameNodeInfo", this);

  infoServer = builder.build();

  infoServer.setAttribute("secondary.name.node", this);
  infoServer.setAttribute("name.system.image", checkpointImage);
  infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
  infoServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
      ImageServlet.class, true);
  infoServer.start();

  LOG.info("Web server init done");

  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    InetSocketAddress httpAddress = infoServer.getConnectorAddress(connIdx++);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
        NetUtils.getHostPortString(httpAddress));
  }

  if (policy.isHttpsEnabled()) {
    InetSocketAddress httpsAddress = infoServer.getConnectorAddress(connIdx);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
        NetUtils.getHostPortString(httpsAddress));
  }

  legacyOivImageDir = conf.get(
      DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY);

  LOG.info("Checkpoint Period   :" + checkpointConf.getPeriod() + " secs "
      + "(" + checkpointConf.getPeriod() / 60 + " min)");
  LOG.info("Log Size Trigger    :" + checkpointConf.getTxnCount() + " txns");
}
Example 8
Source File: SecureDataNodeStarter.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Acquire privileged resources (i.e., the privileged ports) for the data
 * node. The privileged resources consist of the port of the RPC server and
 * the port of HTTP (not HTTPS) server.
 */
@VisibleForTesting
public static SecureResources getSecureResources(Configuration conf)
    throws Exception {
  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  boolean isSecure = UserGroupInformation.isSecurityEnabled();

  // Obtain secure port for data streaming to datanode
  InetSocketAddress streamingAddr  = DataNode.getStreamingAddr(conf);
  int socketWriteTimeout = conf.getInt(
      DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
      HdfsServerConstants.WRITE_TIMEOUT);

  ServerSocket ss = (socketWriteTimeout > 0) ? 
      ServerSocketChannel.open().socket() : new ServerSocket();
  ss.bind(streamingAddr, 0);

  // Check that we got the port we need
  if (ss.getLocalPort() != streamingAddr.getPort()) {
    throw new RuntimeException(
        "Unable to bind on specified streaming port in secure "
            + "context. Needed " + streamingAddr.getPort() + ", got "
            + ss.getLocalPort());
  }

  if (!SecurityUtil.isPrivilegedPort(ss.getLocalPort()) && isSecure) {
    throw new RuntimeException(
      "Cannot start secure datanode with unprivileged RPC ports");
  }

  System.err.println("Opened streaming server at " + streamingAddr);

  // Bind a port for the web server. The code intends to bind HTTP server to
  // privileged port only, as the client can authenticate the server using
  // certificates if they are communicating through SSL.
  final ServerSocketChannel httpChannel;
  if (policy.isHttpEnabled()) {
    httpChannel = ServerSocketChannel.open();
    InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf);
    httpChannel.socket().bind(infoSocAddr);
    InetSocketAddress localAddr = (InetSocketAddress) httpChannel.socket()
      .getLocalSocketAddress();

    if (localAddr.getPort() != infoSocAddr.getPort()) {
      throw new RuntimeException("Unable to bind on specified info port in secure " +
          "context. Needed " + streamingAddr.getPort() + ", got " + ss.getLocalPort());
    }
    System.err.println("Successfully obtained privileged resources (streaming port = "
        + ss + " ) (http listener port = " + localAddr.getPort() +")");

    if (localAddr.getPort() > 1023 && isSecure) {
      throw new RuntimeException(
          "Cannot start secure datanode with unprivileged HTTP ports");
    }
    System.err.println("Opened info server at " + infoSocAddr);
  } else {
    httpChannel = null;
  }

  return new SecureResources(ss, httpChannel);
}