org.apache.hadoop.mapreduce.v2.hs.HistoryContext Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.v2.hs.HistoryContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHsWebServicesJobsQuery.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(3, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #2
Source File: TestHsWebServicesTasks.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #3
Source File: TestHsWebServicesJobsQuery.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(3, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #4
Source File: TestHsWebServicesJobs.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1, false);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #5
Source File: TestHsWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 1, 1);
  JobHistory jobHistoryService = new JobHistory();
  HistoryContext historyContext = (HistoryContext) jobHistoryService;
  webApp = new HsWebApp(historyContext);

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #6
Source File: TestHsWebServicesAttempts.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #7
Source File: TestHsWebServicesTasks.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #8
Source File: TestHsWebServicesJobs.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1, false);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #9
Source File: TestHsWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 1, 1);
  JobHistory jobHistoryService = new JobHistory();
  HistoryContext historyContext = (HistoryContext) jobHistoryService;
  webApp = new HsWebApp(historyContext);

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #10
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void configureServlets() {

  appContext = new MockHistoryContext(0, 1, 2, 1);
  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #11
Source File: JobHistoryServer.java    From XLearning with Apache License 2.0 5 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception {
  Configuration config = new XLearningConfiguration(conf);

  config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

  // This is required for WebApps to use https if enabled.
  XLearningWebAppUtil.initialize(getConfig());

  try {
    doSecureLogin(conf);
  } catch (IOException ie) {
    throw new YarnRuntimeException("History Server Failed to login", ie);
  }

  jobHistoryService = new JobHistory();
  historyContext = (HistoryContext) jobHistoryService;
  stateStore = createStateStore(conf);
  this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore);
  clientService = createHistoryClientService();
  aggLogDelService = new AggregatedLogDeletionService();

  addService(stateStore);
  addService(new HistoryServerSecretManagerService());
  addService(clientService);
  addService(aggLogDelService);
  super.serviceInit(config);
}
Example #12
Source File: TestHsWebServicesAcls.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private static HistoryContext buildHistoryContext(final Configuration conf)
    throws IOException {
  HistoryContext ctx = new MockHistoryContext(1, 1, 1);
  Map<JobId, Job> jobs = ctx.getAllJobs();
  JobId jobId = jobs.keySet().iterator().next();
  Job mockJob = new MockJobForAcls(jobs.get(jobId), conf);
  jobs.put(jobId, mockJob);
  return ctx;
}
Example #13
Source File: TestHsWebServicesJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testJobCountersForKilledJob() throws Exception {
  WebResource r = resource();
  appContext = new MockHistoryContext(0, 1, 1, 1, true);
  injector = Guice.createInjector(new ServletModule() {
    @Override
    protected void configureServlets() {

      webApp = mock(HsWebApp.class);
      when(webApp.name()).thenReturn("hsmockwebapp");

      bind(JAXBContextResolver.class);
      bind(HsWebServices.class);
      bind(GenericExceptionHandler.class);
      bind(WebApp.class).toInstance(webApp);
      bind(AppContext.class).toInstance(appContext);
      bind(HistoryContext.class).toInstance(appContext);
      bind(Configuration.class).toInstance(conf);

      serve("/*").with(GuiceContainer.class);
    }
  });
  
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("counters/")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    WebServicesTestUtils.checkStringMatch("id", MRApps.toString(id),
        info.getString("id"));
    assertTrue("Job shouldn't contain any counters", info.length() == 1);
  }
}
Example #14
Source File: HsWebApp.java    From XLearning with Apache License 2.0 5 votes vote down vote up 
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route(pajoin("/job", APP_ID), HsController.class, "job");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
}
Example #15
Source File: TestHsWebServicesAcls.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static HistoryContext buildHistoryContext(final Configuration conf)
    throws IOException {
  HistoryContext ctx = new MockHistoryContext(1, 1, 1);
  Map<JobId, Job> jobs = ctx.getAllJobs();
  JobId jobId = jobs.keySet().iterator().next();
  Job mockJob = new MockJobForAcls(jobs.get(jobId), conf);
  jobs.put(jobId, mockJob);
  return ctx;
}
Example #16
Source File: HsWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route("/app", HsController.class);
  route(pajoin("/job", JOB_ID), HsController.class, "job");
  route(pajoin("/conf", JOB_ID), HsController.class, "conf");
  route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
  route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleJobCounter");
  route(pajoin("/tasks", JOB_ID, TASK_TYPE), HsController.class, "tasks");
  route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
      HsController.class, "attempts");
  route(pajoin("/task", TASK_ID), HsController.class, "task");
  route(pajoin("/taskcounters", TASK_ID), HsController.class, "taskCounters");
  route(pajoin("/singletaskcounter",TASK_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleTaskCounter");
  route("/about", HsController.class, "about");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
  route(pajoin("/nmlogs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "nmlogs");
}
Example #17
Source File: HsWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void setup() {
  bind(HsWebServices.class);
  bind(JAXBContextResolver.class);
  bind(GenericExceptionHandler.class);
  bind(AppContext.class).toInstance(history);
  bind(HistoryContext.class).toInstance(history);
  route("/", HsController.class);
  route("/app", HsController.class);
  route(pajoin("/job", JOB_ID), HsController.class, "job");
  route(pajoin("/conf", JOB_ID), HsController.class, "conf");
  route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
  route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleJobCounter");
  route(pajoin("/tasks", JOB_ID, TASK_TYPE), HsController.class, "tasks");
  route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
      HsController.class, "attempts");
  route(pajoin("/task", TASK_ID), HsController.class, "task");
  route(pajoin("/taskcounters", TASK_ID), HsController.class, "taskCounters");
  route(pajoin("/singletaskcounter",TASK_ID, COUNTER_GROUP, COUNTER_NAME),
      HsController.class, "singleTaskCounter");
  route("/about", HsController.class, "about");
  route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "logs");
  route(pajoin("/nmlogs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
      CONTAINER_LOG_TYPE), HsController.class, "nmlogs");
}
Example #18
Source File: HistoryClientService.java    From XLearning with Apache License 2.0 5 votes vote down vote up 
public HistoryClientService(HistoryContext history,
                            JHSDelegationTokenSecretManager jhsDTSecretManager) {
  super("HistoryClientService");
  this.history = history;
  this.protocolHandler = new HSClientProtocolHandler();
  this.jhsDTSecretManager = jhsDTSecretManager;
}
Example #19
Source File: TestHsWebServicesJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testJobCountersForKilledJob() throws Exception {
  WebResource r = resource();
  appContext = new MockHistoryContext(0, 1, 1, 1, true);
  injector = Guice.createInjector(new ServletModule() {
    @Override
    protected void configureServlets() {

      webApp = mock(HsWebApp.class);
      when(webApp.name()).thenReturn("hsmockwebapp");

      bind(JAXBContextResolver.class);
      bind(HsWebServices.class);
      bind(GenericExceptionHandler.class);
      bind(WebApp.class).toInstance(webApp);
      bind(AppContext.class).toInstance(appContext);
      bind(HistoryContext.class).toInstance(appContext);
      bind(Configuration.class).toInstance(conf);

      serve("/*").with(GuiceContainer.class);
    }
  });
  
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("counters/")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    WebServicesTestUtils.checkStringMatch("id", MRApps.toString(id),
        info.getString("id"));
    assertTrue("Job shouldn't contain any counters", info.length() == 1);
  }
}
Example #20
Source File: HsWebApp.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public HsWebApp(HistoryContext history) {
  this.history = history;
}
Example #21
Source File: HsWebServices.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Inject
public HsWebServices(final HistoryContext ctx, final Configuration conf,
    final WebApp webapp) {
  this.ctx = ctx;
  this.webapp = webapp;
}
Example #22
Source File: TestHsWebServicesJobConf.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
protected void configureServlets() {

  Path confPath = new Path(testConfDir.toString(),
      MRJobConfig.JOB_CONF_FILE);
  Configuration config = new Configuration();

  FileSystem localFs;
  try {
    localFs = FileSystem.getLocal(config);
    confPath = localFs.makeQualified(confPath);

    OutputStream out = localFs.create(confPath);
    try {
      conf.writeXml(out);
    } finally {
      out.close();
    }
    if (!localFs.exists(confPath)) {
      fail("error creating config file: " + confPath);
    }

  } catch (IOException e) {
    fail("error creating config file: " + e.getMessage());
  }

  appContext = new MockHistoryContext(0, 2, 1, confPath);

  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}
Example #23
Source File: HsWebApp.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public HsWebApp(HistoryContext history) {
  this.history = history;
}
Example #24
Source File: HsWebServices.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Inject
public HsWebServices(final HistoryContext ctx, final Configuration conf,
    final WebApp webapp) {
  this.ctx = ctx;
  this.webapp = webapp;
}
Example #25
Source File: HsWebApp.java    From XLearning with Apache License 2.0 4 votes vote down vote up 
public HsWebApp(HistoryContext history) {
  this.history = history;
}
Example #26
Source File: TestHsWebServicesJobConf.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
protected void configureServlets() {

  Path confPath = new Path(testConfDir.toString(),
      MRJobConfig.JOB_CONF_FILE);
  Configuration config = new Configuration();

  FileSystem localFs;
  try {
    localFs = FileSystem.getLocal(config);
    confPath = localFs.makeQualified(confPath);

    OutputStream out = localFs.create(confPath);
    try {
      conf.writeXml(out);
    } finally {
      out.close();
    }
    if (!localFs.exists(confPath)) {
      fail("error creating config file: " + confPath);
    }

  } catch (IOException e) {
    fail("error creating config file: " + e.getMessage());
  }

  appContext = new MockHistoryContext(0, 2, 1, confPath);

  webApp = mock(HsWebApp.class);
  when(webApp.name()).thenReturn("hsmockwebapp");

  bind(JAXBContextResolver.class);
  bind(HsWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(WebApp.class).toInstance(webApp);
  bind(AppContext.class).toInstance(appContext);
  bind(HistoryContext.class).toInstance(appContext);
  bind(Configuration.class).toInstance(conf);

  serve("/*").with(GuiceContainer.class);
}