org.apache.hadoop.http.HttpServer Java Examples

public void initializeServer() throws IOException {

    String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
    InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
    clusterDaemonServer = RPC.getServer(this, addr.getHostName(),
            addr.getPort(), conf);
    clusterDaemonServer.start();

    // Http server
    String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR,
            "localhost:50143");
    InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
    infoServer = new HttpServer("cb", infoAddr.getHostName(),
            infoAddr.getPort(), infoAddr.getPort() == 0, conf);
    infoServer.setAttribute("cluster.balancer", this);
    infoServer.start();
  }
 

private void startInfoServer() throws IOException {
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(
    java.net.InetAddress.getLocalHost().getCanonicalHostName(),
    0);
  String infoBindAddress = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  infoServer = new HttpServer("jt", infoBindAddress, tmpInfoPort,
      tmpInfoPort == 0, conf);
  infoServer.setAttribute("job.tracker", this);
  infoServer.start();
  this.infoPort = this.infoServer.getPort();

  String hostname =
    java.net.InetAddress.getLocalHost().getCanonicalHostName();
  this.conf.set(
    "mapred.job.tracker.http.address", hostname + ":" + this.infoPort);
  this.conf.setInt("mapred.job.tracker.info.port", this.infoPort);
  this.conf.set("mapred.job.tracker.info.bindAddress", hostname);

  LOG.info("JobTracker webserver: " + this.infoPort);
}
 

protected void initHttpServer(JobConf conf,
    boolean useNettyMapOutputs) throws IOException {

  String infoAddr =
    NetUtils.getServerAddress(conf,
                              "tasktracker.http.bindAddress",
                              "tasktracker.http.port",
                              "mapred.task.tracker.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String httpBindAddress = infoSocAddr.getHostName();
  int httpPort = infoSocAddr.getPort();
  server = new HttpServer("task", httpBindAddress, httpPort,
      httpPort == 0, conf);
  workerThreads = conf.getInt("tasktracker.http.threads", 40);
  server.setThreads(1, workerThreads);
  // let the jsp pages get to the task tracker, config, and other relevant
  // objects
  FileSystem local = FileSystem.getLocal(conf);
  this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
  server.setAttribute("task.tracker", this);
  server.setAttribute("local.file.system", local);
  server.setAttribute("conf", conf);
  server.setAttribute("log", LOG);
  server.setAttribute("localDirAllocator", localDirAllocator);
  server.setAttribute("shuffleServerMetrics", shuffleServerMetrics);
  server.setAttribute(ReconfigurationServlet.
                      CONF_SERVLET_RECONFIGURABLE_PREFIX + "/ttconfchange",
                      TaskTracker.this);
  server.setAttribute("nettyMapOutputHttpPort", nettyMapOutputHttpPort);
  server.addInternalServlet("reconfiguration", "/ttconfchange",
                              ReconfigurationServlet.class);
  server.addInternalServlet(
    "mapOutput", "/mapOutput", MapOutputServlet.class);
  server.addInternalServlet("taskLog", "/tasklog", TaskLogServlet.class);
  server.start();
  this.httpPort = server.getPort();
  checkJettyPort();
}
 

private void startHttpServer() throws IOException {
	String infoAddr = conf.get("mapred.raid.http.address",
			"localhost:50091");
	InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
	this.infoBindAddress = infoSocAddr.getHostName();
	int tmpInfoPort = infoSocAddr.getPort();
	this.infoServer = new HttpServer("raid", this.infoBindAddress,
			tmpInfoPort, tmpInfoPort == 0, conf);
	this.infoServer.setAttribute("raidnode", this);
	this.infoServer.addInternalServlet("corruptfilecounter",
			"/corruptfilecounter", CorruptFileCounterServlet.class);
	this.infoServer.start();
	LOG.info("Web server started at port " + this.infoServer.getPort());
}
 

/**
 * Construct ClusterManager given {@link CoronaConf}
 *
 * @param conf the configuration for the ClusterManager
 * @throws IOException
 */
public ClusterManager(CoronaConf conf) throws IOException {
  this.conf = conf;
  initLegalTypes();

  metrics = new ClusterManagerMetrics(getTypes());

  sessionManager = new SessionManager(this);
  sessionManager.setConf(conf);

  sessionHistoryManager = new SessionHistoryManager();
  sessionHistoryManager.setConf(conf);

  HostsFileReader hostsReader =
      new HostsFileReader(conf.getHostsFile(), conf.getExcludesFile());
  nodeManager = new NodeManager(this, hostsReader);
  nodeManager.setConf(conf);

  sessionNotifier = new SessionNotifier(sessionManager, this, metrics);
  sessionNotifier.setConf(conf);

  scheduler = new Scheduler(nodeManager, sessionManager,
      sessionNotifier, getTypes(), metrics, conf);
  scheduler.start();
  metrics.registerUpdater(scheduler, sessionNotifier);

  InetSocketAddress infoSocAddr =
      NetUtils.createSocketAddr(conf.getClusterManagerHttpAddress());
  infoServer =
      new HttpServer("cm", infoSocAddr.getHostName(), infoSocAddr.getPort(),
                     infoSocAddr.getPort() == 0, conf);
  infoServer.setAttribute("cm", this);
  infoServer.start();

  startTime = clock.getTime();
  hostName = infoSocAddr.getHostName();
  safeMode = false;
}
 

/**
 * Initialize the webserver so that the primary namenode can fetch
 * transaction logs from standby via http.
 */
void initSecondary(Configuration conf) throws IOException {

  nameNodeAddr = avatarNode.getRemoteNamenodeAddress(conf);
  this.primaryNamenode =
      (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID, nameNodeAddr, conf);

  fsName = avatarNode.getRemoteNamenodeHttpName(conf);

  // Initialize other scheduling parameters from the configuration
  checkpointEnabled = conf.getBoolean("fs.checkpoint.enabled", false);
  checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
  checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

  // initialize the webserver for uploading files.
  String infoAddr = 
    NetUtils.getServerAddress(conf,
                              "dfs.secondary.info.bindAddress",
                              "dfs.secondary.info.port",
                              "dfs.secondary.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  infoBindAddress = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort,
      tmpInfoPort == 0, conf);
  infoServer.setAttribute("name.system.image", fsImage);
  this.infoServer.setAttribute("name.conf", conf);
  infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  infoServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = infoServer.getPort();
  conf.set("dfs.secondary.http.address", infoBindAddress + ":" +infoPort);
  LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" +infoPort);
  LOG.warn("Checkpoint Period   :" + checkpointPeriod + " secs " +
           "(" + checkpointPeriod/60 + " min)");
  LOG.warn("Log Size Trigger    :" + checkpointSize + " bytes " +
           "(" + checkpointSize/1024 + " KB)");
}
 

/** {@inheritDoc} */
public void doGet(HttpServletRequest request, HttpServletResponse response
    ) throws ServletException, IOException {
  final UnixUserGroupInformation ugi = getUGI(request);
  final PrintWriter out = response.getWriter();
  final String filename = getFilename(request, response);
  final XMLOutputter xml = new XMLOutputter(out, "UTF-8");
  xml.declaration();

  Configuration daemonConf = (Configuration) getServletContext()
    .getAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE);
  final Configuration conf = (daemonConf == null) ? new Configuration()
    : new Configuration(daemonConf);
  final int socketTimeout = conf.getInt("dfs.socket.timeout", HdfsConstants.READ_TIMEOUT);
  final SocketFactory socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
  UnixUserGroupInformation.saveToConf(conf,
      UnixUserGroupInformation.UGI_PROPERTY_NAME, ugi);
  final ProtocolProxy<ClientProtocol> nnproxy =
    DFSClient.createRPCNamenode(conf);

  try {
    final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
        DataTransferProtocol.DATA_TRANSFER_VERSION,
        filename, nnproxy.getProxy(), nnproxy, socketFactory, socketTimeout);
    MD5MD5CRC32FileChecksum.write(xml, checksum);
  } catch(IOException ioe) {
    new RemoteException(ioe.getClass().getName(), ioe.getMessage()
        ).writeXml(filename, xml);
  }
  xml.endDocument();
}
 

private void startInfoServer(Configuration conf) throws IOException {
  String infoAddr = 
    NetUtils.getServerAddress(conf, 
                            "dfs.datanode.info.bindAddress", 
                            "dfs.datanode.info.port",
                            "dfs.datanode.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String infoHost = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  this.infoServer = new HttpServer("datanode", infoHost, tmpInfoPort,
      tmpInfoPort == 0, conf);
  if (conf.getBoolean("dfs.https.enable", false)) {
    boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
    InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get(
        "dfs.datanode.https.address", infoHost + ":" + 0));
    Configuration sslConf = new Configuration(false);
    sslConf.addResource(conf.get("dfs.https.server.keystore.resource",
        "ssl-server.xml"));
    this.infoServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
    // assume same ssl port for all datanodes
    InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf.get(
        "dfs.datanode.https.address", infoHost + ":" + 50475));
    this.infoServer.setAttribute("datanode.https.port", datanodeSslPort
        .getPort());
  }
  this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class);
  this.infoServer.addInternalServlet(null, "/getFileChecksum/*",
      FileChecksumServlets.GetServlet.class);
  this.infoServer.setAttribute("datanode", this);
  this.infoServer.addServlet(null, "/blockScannerReport", 
                             DataBlockScannerSet.Servlet.class);

  this.infoServer.setAttribute(ReconfigurationServlet.CONF_SERVLET_RECONFIGURABLE_PREFIX +
  CONF_SERVLET_PATH, DataNode.this);
  this.infoServer.addServlet("dnConf", CONF_SERVLET_PATH, ReconfigurationServlet.class);
  this.infoServer.start();
}
 

/**
 * Initialize SecondaryNameNode.
 */
private void initialize(Configuration conf) throws IOException {
  // initiate Java VM metrics
  JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));
  
  // Create connection to the namenode.
  shouldRun = true;
  nameNodeAddr = NameNode.getAddress(conf);

  this.conf = conf;
  this.namenode =
      (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID, nameNodeAddr, conf);

  // initialize checkpoint directories
  fsName = getInfoServer();
  checkpointDirs = FSImage.getCheckpointDirs(conf,
                                "/tmp/hadoop/dfs/namesecondary");
  checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, 
                                "/tmp/hadoop/dfs/namesecondary");    
  checkpointImage = new CheckpointStorage();
  checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

  // Initialize other scheduling parameters from the configuration
  checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
  checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

  // initialize the webserver for uploading files.
  String infoAddr = 
    NetUtils.getServerAddress(conf, 
                              "dfs.secondary.info.bindAddress",
                              "dfs.secondary.info.port",
                              "dfs.secondary.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  infoBindAddress = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort,
      tmpInfoPort == 0, conf);
  infoServer.setAttribute("name.system.image", checkpointImage);
  this.infoServer.setAttribute("name.conf", conf);
  infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  infoServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = infoServer.getPort();
  conf.set("dfs.secondary.http.address", infoBindAddress + ":" +infoPort); 
  LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" +infoPort);
  LOG.warn("Checkpoint Period   :" + checkpointPeriod + " secs " +
           "(" + checkpointPeriod/60 + " min)");
  LOG.warn("Log Size Trigger    :" + checkpointSize + " bytes " +
           "(" + checkpointSize/1024 + " KB)");
}
 

private void startHttpServer(Configuration conf) throws IOException {
  String infoAddr = 
    NetUtils.getServerAddress(conf, "dfs.info.bindAddress", 
                              "dfs.info.port", "dfs.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String infoHost = infoSocAddr.getHostName();
  int infoPort = infoSocAddr.getPort();
  this.httpServer = new HttpServer("hdfs", infoHost, infoPort, 
      infoPort == 0, conf);
  if (conf.getBoolean("dfs.https.enable", false)) {
    boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
    InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get(
        "dfs.https.address", infoHost + ":" + 0));
    Configuration sslConf = new Configuration(false);
    sslConf.addResource(conf.get("dfs.https.server.keystore.resource",
        "ssl-server.xml"));
    this.httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
    // assume same ssl port for all datanodes
    InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf.get(
        "dfs.datanode.https.address", infoHost + ":" + 50475));
    this.httpServer.setAttribute("datanode.https.port", datanodeSslPort
        .getPort());
  }
  this.httpServer.setAttribute("name.node", this);
  this.httpServer.setAttribute("name.node.address", getNameNodeAddress());
  this.httpServer.setAttribute("name.system.image", getFSImage());
  this.httpServer.setAttribute("name.conf", conf);
  this.httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class);
  this.httpServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  this.httpServer.addInternalServlet("listPaths", "/listPaths/*", ListPathsServlet.class);
  this.httpServer.addInternalServlet("data", "/data/*", FileDataServlet.class);
  this.httpServer.addInternalServlet("checksum", "/fileChecksum/*",
      FileChecksumServlets.RedirectServlet.class);
  this.httpServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = this.httpServer.getPort();
  this.httpAddress = new InetSocketAddress(infoHost, infoPort);
  conf.set("dfs.http.address", infoHost + ":" + infoPort);
  LOG.info("Web-server up at: " + infoHost + ":" + infoPort);
}
 

@Override
public void start() {
  try {
    Configuration conf = getConf();
    this.eagerInitListener = new EagerTaskInitializationListener(conf);
    eagerInitListener.setTaskTrackerManager(taskTrackerManager);
    eagerInitListener.start();
    taskTrackerManager.addJobInProgressListener(eagerInitListener);
    taskTrackerManager.addJobInProgressListener(jobListener);
    poolMgr = new PoolManager(conf);
    loadMgr = (LoadManager) ReflectionUtils.newInstance(
        conf.getClass("mapred.fairscheduler.loadmanager", 
            CapBasedLoadManager.class, LoadManager.class), conf);
    loadMgr.setTaskTrackerManager(taskTrackerManager);
    loadMgr.start();
    taskSelector = (TaskSelector) ReflectionUtils.newInstance(
        conf.getClass("mapred.fairscheduler.taskselector", 
            DefaultTaskSelector.class, TaskSelector.class), conf);
    taskSelector.setTaskTrackerManager(taskTrackerManager);
    taskSelector.start();
    Class<?> weightAdjClass = conf.getClass(
        "mapred.fairscheduler.weightadjuster", null);
    if (weightAdjClass != null) {
      weightAdjuster = (WeightAdjuster) ReflectionUtils.newInstance(
          weightAdjClass, conf);
    }
    assignMultiple = conf.getBoolean("mapred.fairscheduler.assignmultiple",
        false);
    sizeBasedWeight = conf.getBoolean("mapred.fairscheduler.sizebasedweight",
        false);
    initialized = true;
    running = true;
    lastUpdateTime = clock.getTime();
    // Start a thread to update deficits every UPDATE_INTERVAL
    if (runBackgroundUpdates)
      new UpdateThread().start();
    // Register servlet with JobTracker's Jetty server
    if (taskTrackerManager instanceof JobTracker) {
      JobTracker jobTracker = (JobTracker) taskTrackerManager;
      HttpServer infoServer = jobTracker.infoServer;
      infoServer.setAttribute("scheduler", this);
      infoServer.addServlet("scheduler", "/scheduler",
          FairSchedulerServlet.class);
    }
  } catch (Exception e) {
    // Can't load one of the managers - crash the JobTracker now while it is
    // starting up so that the user notices.
    throw new RuntimeException("Failed to start FairScheduler", e);
  }
  LOG.info("Successfully configured FairScheduler");
}
 

public void testDynamicLogLevel() throws Exception {
  String logName = TestLogLevel.class.getName();
  Log testlog = LogFactory.getLog(logName);

  //only test Log4JLogger
  if (testlog instanceof Log4JLogger) {
    Logger log = ((Log4JLogger)testlog).getLogger();
    log.debug("log.debug1");
    log.info("log.info1");
    log.error("log.error1");
    assertTrue(!Level.ERROR.equals(log.getEffectiveLevel()));

    HttpServer server = new HttpServer("..", "localhost", 22222, true);
    server.start();
    int port = server.getPort();

    //servlet
    URL url = new URL("http://localhost:" + port
        + "/logLevel?log=" + logName + "&level=" + Level.ERROR);
    out.println("*** Connecting to " + url);
    URLConnection connection = url.openConnection();
    connection.connect();

    BufferedReader in = new BufferedReader(new InputStreamReader(
        connection.getInputStream()));
    for(String line; (line = in.readLine()) != null; out.println(line));
    in.close();

    log.debug("log.debug2");
    log.info("log.info2");
    log.error("log.error2");
    assertTrue(Level.ERROR.equals(log.getEffectiveLevel()));

    //command line
    String[] args = {"-setlevel", "localhost:"+port, logName,""+Level.DEBUG};
    LogLevel.main(args);
    log.debug("log.debug3");
    log.info("log.info3");
    log.error("log.error3");
    assertTrue(Level.DEBUG.equals(log.getEffectiveLevel()));
  }
  else {
    out.println(testlog.getClass() + " not tested.");
  }
}
 

/**
 * Start with the local machine name, and the default JobTracker
 */
public TaskTracker(JobConf conf) throws IOException {
  originalConf = conf;
  //maxCurrentMapTasks = conf.getInt(
  // "mapred.tasktracker.map.tasks.maximum", 2);
  maxCurrentCPUMapTasks = conf.getInt(
  							"mapred.tasktracker.map.cpu.tasks.maximum", 2);
  maxCurrentGPUMapTasks = conf.getInt(
  							"mapred.tasktracker.map.gpu.tasks.maximum", 0);
  maxCurrentMapTasks = maxCurrentCPUMapTasks + maxCurrentGPUMapTasks;
  maxCurrentReduceTasks = conf.getInt(
                "mapred.tasktracker.reduce.tasks.maximum", 2);
  this.jobTrackAddr = JobTracker.getAddress(conf);
  String infoAddr = 
    NetUtils.getServerAddress(conf,
                              "tasktracker.http.bindAddress", 
                              "tasktracker.http.port",
                              "mapred.task.tracker.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String httpBindAddress = infoSocAddr.getHostName();
  int httpPort = infoSocAddr.getPort();
  this.server = new HttpServer("task", httpBindAddress, httpPort,
      httpPort == 0, conf);
  workerThreads = conf.getInt("tasktracker.http.threads", 40);
  this.shuffleServerMetrics = new ShuffleServerMetrics(conf);
  server.setThreads(1, workerThreads);
  // let the jsp pages get to the task tracker, config, and other relevant
  // objects
  FileSystem local = FileSystem.getLocal(conf);
  this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
  server.setAttribute("task.tracker", this);
  server.setAttribute("local.file.system", local);
  server.setAttribute("conf", conf);
  server.setAttribute("log", LOG);
  server.setAttribute("localDirAllocator", localDirAllocator);
  server.setAttribute("shuffleServerMetrics", shuffleServerMetrics);
  server.addInternalServlet("mapOutput", "/mapOutput", MapOutputServlet.class);
  server.addInternalServlet("taskLog", "/tasklog", TaskLogServlet.class);
  server.start();
  this.httpPort = server.getPort();
  checkJettyPort(httpPort);
  initialize();
}
 

/**
 * Initialize SecondaryNameNode.
 */
private void initialize(Configuration conf) throws IOException {
  // initiate Java VM metrics
  JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));
  
  // Create connection to the namenode.
  shouldRun = true;
  nameNodeAddr = NameNode.getAddress(conf);

  this.conf = conf;
  this.namenode =
      (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID, nameNodeAddr, conf);

  // initialize checkpoint directories
  fsName = getInfoServer();
  checkpointDirs = FSImage.getCheckpointDirs(conf,
                                "/tmp/hadoop/dfs/namesecondary");
  checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, 
                                "/tmp/hadoop/dfs/namesecondary");    
  checkpointImage = new CheckpointStorage(conf);
  checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

  // Initialize other scheduling parameters from the configuration
  checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
  checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

  // initialize the webserver for uploading files.
  String infoAddr = 
    NetUtils.getServerAddress(conf, 
                              "dfs.secondary.info.bindAddress",
                              "dfs.secondary.info.port",
                              "dfs.secondary.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  infoBindAddress = infoSocAddr.getHostName();
  int tmpInfoPort = infoSocAddr.getPort();
  infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort,
      tmpInfoPort == 0, conf);
  infoServer.setAttribute("name.system.image", checkpointImage);
  this.infoServer.setAttribute("name.conf", conf);
  infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  infoServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = infoServer.getPort();
  conf.set("dfs.secondary.http.address", infoBindAddress + ":" +infoPort); 
  LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" +infoPort);
  LOG.warn("Checkpoint Period   :" + checkpointPeriod + " secs " +
           "(" + checkpointPeriod/60 + " min)");
  LOG.warn("Log Size Trigger    :" + checkpointSize + " bytes " +
           "(" + checkpointSize/1024 + " KB)");
}
 

private void startHttpServer(Configuration conf) throws IOException {
  String infoAddr =
    NetUtils.getServerAddress(conf, "dfs.info.bindAddress",
                              "dfs.info.port", "dfs.http.address");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String infoHost = infoSocAddr.getHostName();
  int infoPort = infoSocAddr.getPort();
  this.httpServer = new HttpServer("hdfs", infoHost, infoPort,
      infoPort == 0, conf);
  if (conf.getBoolean("dfs.https.enable", false)) {
    boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
    InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(conf.get(
        "dfs.https.address", infoHost + ":" + 0));
    Configuration sslConf = new Configuration(false);
    sslConf.addResource(conf.get("dfs.https.server.keystore.resource",
        "ssl-server.xml"));
    this.httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
    // assume same ssl port for all datanodes
    InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf.get(
        "dfs.datanode.https.address", infoHost + ":" + 50475));
    this.httpServer.setAttribute("datanode.https.port", datanodeSslPort
        .getPort());
  }
  this.httpServer.setAttribute("name.node", this);
  this.httpServer.setAttribute("name.node.address", getNameNodeAddress());
  this.httpServer.setAttribute("name.system.image", getFSImage());
  this.httpServer.setAttribute("name.conf", conf);
  this.httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class);
  this.httpServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
  this.httpServer.addInternalServlet("listPaths", "/listPaths/*", ListPathsServlet.class);
  this.httpServer.addInternalServlet("data", "/data/*", FileDataServlet.class);
  this.httpServer.addInternalServlet("checksum", "/fileChecksum/*",
      FileChecksumServlets.RedirectServlet.class);
  this.httpServer.addInternalServlet("namenodeMXBean", "/namenodeMXBean", 
      NameNodeMXBeanServlet.class);
  httpServer.setAttribute(ReconfigurationServlet.
                          CONF_SERVLET_RECONFIGURABLE_PREFIX +
                          CONF_SERVLET_PATH, NameNode.this);
  httpServer.addInternalServlet("nnconfchange", CONF_SERVLET_PATH,
                                ReconfigurationServlet.class);

  this.httpServer.start();

  // The web-server port can be ephemeral... ensure we have the correct info
  infoPort = this.httpServer.getPort();
  this.httpAddress = new InetSocketAddress(infoHost, infoPort);
  conf.set("dfs.http.address", infoHost + ":" + infoPort);
  LOG.info("Web-server up at: " + infoHost + ":" + infoPort);
}
 

public ProxyJobTracker(CoronaConf conf) throws IOException {
  this.conf = conf;
  fs = FileSystem.get(conf);
  String infoAddr =
    conf.get("mapred.job.tracker.corona.proxyaddr", "0.0.0.0:0");
  InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
  String infoBindAddress = infoSocAddr.getHostName();
  int port = infoSocAddr.getPort();
  LOCALMACHINE = infoBindAddress;
  startTime = getClock().getTime();

  CoronaConf coronaConf = new CoronaConf(conf);
  InetSocketAddress rpcSockAddr = NetUtils.createSocketAddr(
    coronaConf.getProxyJobTrackerAddress());
  rpcServer = RPC.getServer(
    this,
    rpcSockAddr.getHostName(),
    rpcSockAddr.getPort(),
    conf.getInt("corona.proxy.job.tracker.handler.count", 10),
    false,
    conf);
  rpcServer.start();

  LOG.info("ProxyJobTracker RPC Server up at " +
    rpcServer.getListenerAddress());

  infoServer = new HttpServer("proxyjt", infoBindAddress, port,
                              port == 0, conf);
  infoServer.setAttribute("proxy.job.tracker", this);
  infoServer.setAttribute("conf", conf);
  infoServer.addServlet("proxy", "/proxy",
                        ProxyJobTrackerServlet.class);
  // initialize history parameters.
  JobConf jobConf = new JobConf(conf);
  boolean historyInitialized = JobHistory.init(
    this, jobConf, this.LOCALMACHINE, this.startTime);
  if (historyInitialized) {
    JobHistory.initDone(jobConf, fs);
    String historyLogDir =
        JobHistory.getCompletedJobHistoryLocation().toString();
    FileSystem historyFS = new Path(historyLogDir).getFileSystem(conf);
    infoServer.setAttribute("historyLogDir", historyLogDir);
    infoServer.setAttribute("fileSys", historyFS);
  }
  infoServer.start();
  LOCALPORT = infoServer.getPort();

  context = MetricsUtil.getContext("mapred");
  metricsRecord = MetricsUtil.createRecord(context, "proxyjobtracker");
  context.registerUpdater(this);

  expireUnusedFilesInCache = new ExpireUnusedFilesInCache(
    conf, getClock(), new Path(getSystemDir()), fs);

  sessionHistoryManager = new SessionHistoryManager();
  sessionHistoryManager.setConf(conf);

  try {
    String target = conf.getProxyJobTrackerThriftAddress();
    InetSocketAddress addr = NetUtils.createSocketAddr(target);
    LOG.info("Trying to start the Thrift Server at: " + target);
    ServerSocket serverSocket = new ServerSocket(addr.getPort());
    server = new TServerThread(
      TFactoryBasedThreadPoolServer.createNewServer(
        new CoronaProxyJobTrackerService.Processor(this),
        serverSocket,
        5000));
    server.start();
    LOG.info("Thrift server started on: " + target);
  } catch (IOException e) {
    LOG.info("Exception while starting the Thrift Server on CPJT: ", e);
  }
}
 

public HttpServer getInfoServer() {
	return infoServer;
}
 

public void testDynamicLogLevel() throws Exception {
  String logName = TestLogLevel.class.getName();
  Log testlog = LogFactory.getLog(logName);

  //only test Log4JLogger
  if (testlog instanceof Log4JLogger) {
    Logger log = ((Log4JLogger)testlog).getLogger();
    log.debug("log.debug1");
    log.info("log.info1");
    log.error("log.error1");
    assertTrue(!Level.ERROR.equals(log.getEffectiveLevel()));

    HttpServer server = new HttpServer("..", "localhost", 22222, true);
    server.start();
    int port = server.getPort();

    //servlet
    URL url = new URL("http://localhost:" + port
        + "/logLevel?log=" + logName + "&level=" + Level.ERROR);
    out.println("*** Connecting to " + url);
    URLConnection connection = url.openConnection();
    connection.connect();

    BufferedReader in = new BufferedReader(new InputStreamReader(
        connection.getInputStream()));
    for(String line; (line = in.readLine()) != null; out.println(line));
    in.close();

    log.debug("log.debug2");
    log.info("log.info2");
    log.error("log.error2");
    assertTrue(Level.ERROR.equals(log.getEffectiveLevel()));

    //command line
    String[] args = {"-setlevel", "localhost:"+port, logName,""+Level.DEBUG};
    LogLevel.main(args);
    log.debug("log.debug3");
    log.info("log.info3");
    log.error("log.error3");
    assertTrue(Level.DEBUG.equals(log.getEffectiveLevel()));
  }
  else {
    out.println(testlog.getClass() + " not tested.");
  }
}