Java Code Examples for org.apache.hadoop.net.NetUtils#createSocketAddr()

/**
 * Setup a session with the specified datanode
 */
static ClientDatanodeProtocol createClientDatanodeProtocolProxy (
    DatanodeInfo datanodeid, Configuration conf) throws IOException {
  InetSocketAddress addr = NetUtils.createSocketAddr(
    datanodeid.getHost() + ":" + datanodeid.getIpcPort());
  if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
    ClientDatanodeProtocol.LOG.info("ClientDatanodeProtocol addr=" + addr);
  }
  try {
    return (ClientDatanodeProtocol)RPC.getProxy(ClientDatanodeProtocol.class,
      ClientDatanodeProtocol.versionID, addr, conf);
  } catch (RPC.VersionMismatch e) {
    long clientVersion = e.getClientVersion();
    long datanodeVersion = e.getServerVersion();
    if (clientVersion > datanodeVersion &&
        !ProtocolCompatible.isCompatibleClientDatanodeProtocol(
            clientVersion, datanodeVersion)) {
      throw new RPC.VersionIncompatible(
          ClientDatanodeProtocol.class.getName(), clientVersion, datanodeVersion);
    }
    return (ClientDatanodeProtocol)e.getProxy();
  }
}
 

/**
 * Get the socket address for <code>hostProperty</code> as a
 * <code>InetSocketAddress</code>. If <code>hostProperty</code> is
 * <code>null</code>, <code>addressProperty</code> will be used. This
 * is useful for cases where we want to differentiate between host
 * bind address and address clients should use to establish connection.
 *
 * @param hostProperty bind host property name.
 * @param addressProperty address property name.
 * @param defaultAddressValue the default value
 * @param defaultPort the default port
 * @return InetSocketAddress
 */
public InetSocketAddress getSocketAddr(
    String hostProperty,
    String addressProperty,
    String defaultAddressValue,
    int defaultPort) {

  InetSocketAddress bindAddr = getSocketAddr(
    addressProperty, defaultAddressValue, defaultPort);

  final String host = get(hostProperty);

  if (host == null || host.isEmpty()) {
    return bindAddr;
  }

  return NetUtils.createSocketAddr(
      host, bindAddr.getPort(), hostProperty);
}
 

private void initialize(Configuration conf) throws IOException {
  sslAddr = getSslAddr(conf);
  String nn = conf.get("hdfsproxy.dfs.namenode.address");
  if (nn == null)
    throw new IOException("HDFS NameNode address is not specified");
  InetSocketAddress nnAddr = NetUtils.createSocketAddr(nn);
  LOG.info("HDFS NameNode is at: " + nnAddr.getHostName() + ":" + nnAddr.getPort());

  Configuration sslConf = new Configuration(false);
  sslConf.addResource(conf.get("hdfsproxy.https.server.keystore.resource",
      "ssl-server.xml"));
  // unit testing
  sslConf.set("proxy.http.test.listener.addr",
              conf.get("proxy.http.test.listener.addr"));

  this.server = new ProxyHttpServer(sslAddr, sslConf);
  this.server.setAttribute("proxy.https.port", server.getPort());
  this.server.setAttribute("name.node.address", nnAddr);
  this.server.setAttribute("name.conf", new Configuration());
  this.server.addGlobalFilter("ProxyFilter", ProxyFilter.class.getName(), null);
  this.server.addServlet("listPaths", "/listPaths/*", ProxyListPathsServlet.class);
  this.server.addServlet("data", "/data/*", ProxyFileDataServlet.class);
  this.server.addServlet("streamFile", "/streamFile/*", ProxyStreamFile.class);
}
 

/**
 * Create a socket for a write pipeline
 * @param first the first datanode 
 * @param length the pipeline length
 * @param client client
 * @return the socket connected to the first datanode
 */
static Socket createSocketForPipeline(final DatanodeInfo first,
    final int length, final DFSClient client) throws IOException {
  final String dnAddr = first.getXferAddr(
      client.getConf().connectToDnViaHostname);
  if (DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
  }
  final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
  final Socket sock = client.socketFactory.createSocket();
  final int timeout = client.getDatanodeReadTimeout(length);
  NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().socketTimeout);
  sock.setSoTimeout(timeout);
  sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
  if(DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize());
  }
  return sock;
}
 

private static ApplicationClientProtocol getRmClient(Token<?> token,
    Configuration conf) throws IOException {
  String[] services = token.getService().toString().split(",");
  for (String service : services) {
    InetSocketAddress addr = NetUtils.createSocketAddr(service);
    if (localSecretManager != null) {
      // return null if it's our token
      if (localServiceAddress.getAddress().isAnyLocalAddress()) {
        if (NetUtils.isLocalAddress(addr.getAddress()) &&
            addr.getPort() == localServiceAddress.getPort()) {
          return null;
        }
      } else if (addr.equals(localServiceAddress)) {
        return null;
      }
    }
  }
  return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
}
 

static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
    DatanodeID datanodeid, Configuration conf, int socketTimeout,
    boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
  final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
  InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
  }
  
  // Since we're creating a new UserGroupInformation here, we know that no
  // future RPC proxies will be able to re-use the same connection. And
  // usages of this proxy tend to be one-off calls.
  //
  // This is a temporary fix: callers should really achieve this by using
  // RPC.stopProxy() on the resulting object, but this is currently not
  // working in trunk. See the discussion on HDFS-1965.
  Configuration confWithNoIpcIdle = new Configuration(conf);
  confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
      .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);

  UserGroupInformation ticket = UserGroupInformation
      .createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
  ticket.addToken(locatedBlock.getBlockToken());
  return createClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle,
      NetUtils.getDefaultSocketFactory(conf), socketTimeout);
}
 

void start() throws IOException {
  final InetSocketAddress httpAddr = getHttpAddress(conf);

  final String httpsAddrString = conf.get(
      NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY,
      NfsConfigKeys.NFS_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "nfs3",
      NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY,
      NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY);

  this.httpServer = builder.build();
  this.httpServer.start();
  
  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    infoPort = httpServer.getConnectorAddress(connIdx++).getPort();
  }

  if (policy.isHttpsEnabled()) {
    infoSecurePort = httpServer.getConnectorAddress(connIdx).getPort();
  }
}
 

private void reportJobStats() {
  if (job == null) {
    return;
  }
  Counters jobCounters = job.getCounters();
  JobStats jobStats = job.getJobStats();
  String pool = null;
  if (sessionDriver != null) {
    pool = PoolInfo.createStringFromPoolInfo(sessionDriver.getPoolInfo());
  }
  try {
    CoronaConf coronaConf = new CoronaConf(conf);
    InetSocketAddress aggregatorAddr = NetUtils.createSocketAddr(
      coronaConf.getProxyJobTrackerAddress());
    long timeout = 5000; // Can make configurable later.
    CoronaJobAggregator aggregator = RPC.waitForProxy(
      CoronaJobAggregator.class,
      CoronaJobAggregator.versionID,
      aggregatorAddr,
      conf,
      timeout);
    LOG.info("Reporting job stats with jobId=" + jobId +
      ", pool=" + pool + ", jobStats=" + jobStats + ", " +
      "jobCounters=" + jobCounters);
    aggregator.reportJobStats(jobId.toString(), pool, jobStats, jobCounters);
  } catch (IOException e) {
    LOG.warn("Ignoring error in reportJobStats ", e);
  }
}
 

private static InetSocketAddress getAddress(Configuration conf) {
  String addr = conf.get(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY,
      DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT);
  return NetUtils.createSocketAddr(addr,
      DFSConfigKeys.DFS_JOURNALNODE_HTTP_PORT_DEFAULT,
      DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY);
}
 

private static String getHostnameForSpnegoPrincipal(Configuration conf) {
  String addr = conf.getTrimmed(DFS_DATANODE_HTTP_ADDRESS_KEY, null);
  if (addr == null) {
    addr = conf.getTrimmed(DFS_DATANODE_HTTPS_ADDRESS_KEY,
                           DFS_DATANODE_HTTPS_ADDRESS_DEFAULT);
  }
  InetSocketAddress inetSocker = NetUtils.createSocketAddr(addr);
  return inetSocker.getHostString();
}
 

@Override
protected InetSocketAddress getServiceRpcServerAddress(Configuration conf) {
  String addr = conf.getTrimmed(BN_SERVICE_RPC_ADDRESS_KEY);
  if (addr == null || addr.isEmpty()) {
    return null;
  }
  return NetUtils.createSocketAddr(addr);
}
 

static InetSocketAddress getAddress(Configuration conf) {
  String addr = conf.get(
      DFSConfigKeys.DFS_JOURNALNODE_RPC_ADDRESS_KEY,
      DFSConfigKeys.DFS_JOURNALNODE_RPC_ADDRESS_DEFAULT);
  return NetUtils.createSocketAddr(addr, 0,
      DFSConfigKeys.DFS_JOURNALNODE_RPC_ADDRESS_KEY);
}
 

/**
 * Get the socket address for <code>name</code> property as a
 * <code>InetSocketAddress</code>. On a HA cluster,
 * this fetches the address corresponding to the RM identified by
 * {@link #RM_HA_ID}.
 * @param name property name.
 * @param defaultAddress the default value
 * @param defaultPort the default port
 * @return InetSocketAddress
 */
@Override
public InetSocketAddress getSocketAddr(
    String name, String defaultAddress, int defaultPort) {
  String address;
  if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) {
    address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this);
  } else {
    address = get(name, defaultAddress);
  }
  return NetUtils.createSocketAddr(address, defaultPort, name);
}
 

private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 

public int genericRefresh(String[] argv, int i) throws IOException {
  String hostport = argv[i++];
  String identifier = argv[i++];
  String[] args = Arrays.copyOfRange(argv, i, argv.length);

  // Get the current configuration
  Configuration conf = getConf();

  // for security authorization
  // server principal for this call
  // should be NN's one.
  conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
    conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, ""));

  // Create the client
  Class<?> xface = GenericRefreshProtocolPB.class;
  InetSocketAddress address = NetUtils.createSocketAddr(hostport);
  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

  RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
  GenericRefreshProtocolPB proxy = (GenericRefreshProtocolPB)
    RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
      ugi, conf, NetUtils.getDefaultSocketFactory(conf), 0);

  Collection<RefreshResponse> responses = null;
  try (GenericRefreshProtocolClientSideTranslatorPB xlator =
      new GenericRefreshProtocolClientSideTranslatorPB(proxy);) {
    // Refresh
    responses = xlator.refresh(identifier, args);

    int returnCode = 0;

    // Print refresh responses
    System.out.println("Refresh Responses:\n");
    for (RefreshResponse response : responses) {
      System.out.println(response.toString());

      if (returnCode == 0 && response.getReturnCode() != 0) {
        // This is the first non-zero return code, so we should return this
        returnCode = response.getReturnCode();
      } else if (returnCode != 0 && response.getReturnCode() != 0) {
        // Then now we have multiple non-zero return codes,
        // so we merge them into -1
        returnCode = - 1;
      }
    }
    return returnCode;
  } finally {
    if (responses == null) {
      System.out.println("Failed to get response.\n");
      return -1;
    }
  }
}
 

/**
 * Use {@link NetUtils#createSocketAddr(String)} instead.
 */
@Deprecated
public static InetSocketAddress createSocketAddr(String target) {
  return NetUtils.createSocketAddr(target);
}
 

public SentryService(Configuration conf) {
  this.conf = conf;
  int port = conf
      .getInt(ServerConfig.RPC_PORT, ServerConfig.RPC_PORT_DEFAULT);
  if (port == 0) {
    port = findFreePort();
    conf.setInt(ServerConfig.RPC_PORT, port);
  }
  this.address = NetUtils.createSocketAddr(
      conf.get(ServerConfig.RPC_ADDRESS, ServerConfig.RPC_ADDRESS_DEFAULT),
      port);
  LOGGER.info("Configured on address " + address);
  kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
      conf.get(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_KERBEROS).trim());
  maxThreads = conf.getInt(ServerConfig.RPC_MAX_THREADS,
      ServerConfig.RPC_MAX_THREADS_DEFAULT);
  minThreads = conf.getInt(ServerConfig.RPC_MIN_THREADS,
      ServerConfig.RPC_MIN_THREADS_DEFAULT);
  maxMessageSize = conf.getLong(ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE,
      ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE_DEFAULT);
  if (kerberos) {
    // Use Hadoop libraries to translate the _HOST placeholder with actual hostname
    try {
      String rawPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL), ServerConfig.PRINCIPAL + " is required");
      principal = SecurityUtil.getServerPrincipal(rawPrincipal, address.getAddress());
    } catch(IOException io) {
      throw new RuntimeException("Can't translate kerberos principal'", io);
    }
    LOGGER.info("Using kerberos principal: " + principal);

    principalParts = SaslRpcServer.splitKerberosName(principal);
    Preconditions.checkArgument(principalParts.length == 3,
        "Kerberos principal should have 3 parts: " + principal);
    keytab = Preconditions.checkNotNull(conf.get(ServerConfig.KEY_TAB),
        ServerConfig.KEY_TAB + " is required");
    File keytabFile = new File(keytab);
    Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
        "Keytab " + keytab + " does not exist or is not readable.");
  } else {
    principal = null;
    principalParts = null;
    keytab = null;
  }
  serviceExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
    private int count = 0;

    @Override
    public Thread newThread(Runnable r) {
      return new Thread(r, SentryService.class.getSimpleName() + "-"
          + (count++));
    }
  });
  webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT);
  status = Status.NOT_STARTED;
}
 

private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
                       OutputStream fos) throws Exception {
  int failures = 0;
  InetSocketAddress targetAddr = null;
  TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
  BlockReader blockReader = null; 
  ExtendedBlock block = lblock.getBlock(); 

  while (blockReader == null) {
    DatanodeInfo chosenNode;
    
    try {
      chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
      targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
    }  catch (IOException ie) {
      if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
        throw new IOException("Could not obtain block " + lblock, ie);
      }
      LOG.info("Could not obtain block from any node:  " + ie);
      try {
        Thread.sleep(10000);
      }  catch (InterruptedException iex) {
      }
      deadNodes.clear();
      failures++;
      continue;
    }
    try {
      String file = BlockReaderFactory.getFileName(targetAddr,
          block.getBlockPoolId(), block.getBlockId());
      blockReader = new BlockReaderFactory(dfs.getConf()).
          setFileName(file).
          setBlock(block).
          setBlockToken(lblock.getBlockToken()).
          setStartOffset(0).
          setLength(-1).
          setVerifyChecksum(true).
          setClientName("fsck").
          setDatanodeInfo(chosenNode).
          setInetSocketAddress(targetAddr).
          setCachingStrategy(CachingStrategy.newDropBehind()).
          setClientCacheContext(dfs.getClientContext()).
          setConfiguration(namenode.conf).
          setRemotePeerFactory(new RemotePeerFactory() {
            @Override
            public Peer newConnectedPeer(InetSocketAddress addr,
                Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
                throws IOException {
              Peer peer = null;
              Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
              try {
                s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
                s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
                peer = TcpPeerServer.peerFromSocketAndKey(
                      dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
                      blockToken, datanodeId);
              } finally {
                if (peer == null) {
                  IOUtils.closeQuietly(s);
                }
              }
              return peer;
            }
          }).
          build();
    }  catch (IOException ex) {
      // Put chosen node into dead list, continue
      LOG.info("Failed to connect to " + targetAddr + ":" + ex);
      deadNodes.add(chosenNode);
    }
  }
  byte[] buf = new byte[1024];
  int cnt = 0;
  boolean success = true;
  long bytesRead = 0;
  try {
    while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
      fos.write(buf, 0, cnt);
      bytesRead += cnt;
    }
    if ( bytesRead != block.getNumBytes() ) {
      throw new IOException("Recorded block size is " + block.getNumBytes() + 
                            ", but datanode returned " +bytesRead+" bytes");
    }
  } catch (Exception e) {
    LOG.error("Error reading block", e);
    success = false;
  } finally {
    blockReader.close();
  }
  if (!success) {
    throw new Exception("Could not copy block data for " + lblock.getBlock());
  }
}
 

/**
 * Use {@link NetUtils#createSocketAddr(String)} instead.
 */
@Deprecated
public static InetSocketAddress createSocketAddr(String target) {
  return NetUtils.createSocketAddr(target);
}
 

/**
 * Get the socket address for <code>name</code> property as a
 * <code>InetSocketAddress</code>.
 * @param name property name.
 * @param defaultAddress the default value
 * @param defaultPort the default port
 * @return InetSocketAddress
 */
public InetSocketAddress getSocketAddr(
    String name, String defaultAddress, int defaultPort) {
  final String address = getTrimmed(name, defaultAddress);
  return NetUtils.createSocketAddr(address, defaultPort, name);
}