org.apache.hadoop.service.Service Java Examples

/**
 * Show that if the service failed during an init
 * operation, stop was called.
 */

@Test
public void testStopFailingInitAndStop() throws Throwable {
  BreakableService svc = new BreakableService(true, false, true);
  svc.registerServiceListener(new LoggingStateChangeListener());
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    assertEquals(Service.STATE.INITED, e.state);
  }
  //the service state is stopped
  assertServiceStateStopped(svc);
  assertEquals(Service.STATE.INITED, svc.getFailureState());

  Throwable failureCause = svc.getFailureCause();
  assertNotNull("Null failure cause in " + svc, failureCause);
  BreakableService.BrokenLifecycleEvent cause =
    (BreakableService.BrokenLifecycleEvent) failureCause;
  assertNotNull("null state in " + cause + " raised by " + svc, cause.state);
  assertEquals(Service.STATE.INITED, cause.state);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  // init timeline services first
  timelineStore = createTimelineStore(conf);
  addIfService(timelineStore);
  secretManagerService = createTimelineDelegationTokenSecretManagerService(conf);
  addService(secretManagerService);
  timelineDataManager = createTimelineDataManager(conf);
  addService(timelineDataManager);

  // init generic history service afterwards
  aclsManager = createApplicationACLsManager(conf);
  historyManager = createApplicationHistoryManager(conf);
  ahsClientService = createApplicationHistoryClientService(historyManager);
  addService(ahsClientService);
  addService((Service) historyManager);

  DefaultMetricsSystem.initialize("ApplicationHistoryServer");
  JvmMetrics.initSingleton("ApplicationHistoryServer", null);
  super.serviceInit(conf);
}
 

/**
 * Show that if the service failed during an init
 * operation, stop was called.
 */

@Test
public void testStopFailingInitAndStop() throws Throwable {
  BreakableService svc = new BreakableService(true, false, true);
  svc.registerServiceListener(new LoggingStateChangeListener());
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    assertEquals(Service.STATE.INITED, e.state);
  }
  //the service state is stopped
  assertServiceStateStopped(svc);
  assertEquals(Service.STATE.INITED, svc.getFailureState());

  Throwable failureCause = svc.getFailureCause();
  assertNotNull("Null failure cause in " + svc, failureCause);
  BreakableService.BrokenLifecycleEvent cause =
    (BreakableService.BrokenLifecycleEvent) failureCause;
  assertNotNull("null state in " + cause + " raised by " + svc, cause.state);
  assertEquals(Service.STATE.INITED, cause.state);
}
 

/**
 * Tests that the YarnClient is only shut down if it is not shared.
 */
@Test
public void testYarnClientShutDown() {
	YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor();

	yarnClusterDescriptor.close();

	assertTrue(yarnClient.isInState(Service.STATE.STARTED));

	final YarnClient closableYarnClient = YarnClient.createYarnClient();
	closableYarnClient.init(yarnConfiguration);
	closableYarnClient.start();

	yarnClusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(
			temporaryFolder.getRoot().getAbsolutePath(),
			new Configuration(),
			yarnConfiguration,
			closableYarnClient,
			false);

	yarnClusterDescriptor.close();

	assertTrue(closableYarnClient.isInState(Service.STATE.STOPPED));
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

@Test
public void testPolicyInitializeAfterSchedulerInitialized() {
  @SuppressWarnings("resource")
  MockRM rm = new MockRM(conf);
  rm.init(conf);
  
  // ProportionalCapacityPreemptionPolicy should be initialized after
  // CapacityScheduler initialized. We will 
  // 1) find SchedulingMonitor from RMActiveService's service list, 
  // 2) check if ResourceCalculator in policy is null or not. 
  // If it's not null, we can come to a conclusion that policy initialized
  // after scheduler got initialized
  for (Service service : rm.getRMActiveService().getServices()) {
    if (service instanceof SchedulingMonitor) {
      ProportionalCapacityPreemptionPolicy policy =
          (ProportionalCapacityPreemptionPolicy) ((SchedulingMonitor) service)
              .getSchedulingEditPolicy();
      assertNotNull(policy.getResourceCalculator());
      return;
    }
  }
  
  fail("Failed to find SchedulingMonitor service, please check what happened");
}
 

public void waitForState(Service.STATE finalState) throws Exception {
  if (finalState == Service.STATE.STOPPED) {
     Assert.assertTrue("Timeout while waiting for MRApp to stop",
         waitForServiceToStop(20 * 1000));
  } else {
    int timeoutSecs = 0;
    while (!finalState.equals(getServiceState()) && timeoutSecs++ < 20) {
      System.out.println("MRApp State is : " + getServiceState()
          + " Waiting for state : " + finalState);
      Thread.sleep(500);
    }
    System.out.println("MRApp State is : " + getServiceState());
    Assert.assertEquals("MRApp state is not correct (timedout)", finalState,
        getServiceState());
  }
}
 

/**
 * Walk the {@link BreakableService} through it's lifecycle, 
 * more to verify that service's counters work than anything else
 * @throws Throwable if necessary
 */
@Test
public void testWalkthrough() throws Throwable {

  BreakableService svc = new BreakableService();
  assertServiceStateCreated(svc);
  assertStateCount(svc, Service.STATE.NOTINITED, 1);
  assertStateCount(svc, Service.STATE.INITED, 0);
  assertStateCount(svc, Service.STATE.STARTED, 0);
  assertStateCount(svc, Service.STATE.STOPPED, 0);
  svc.init(new Configuration());
  assertServiceStateInited(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  svc.start();
  assertServiceStateStarted(svc);
  assertStateCount(svc, Service.STATE.STARTED, 1);
  svc.stop();
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

/**
 * Show that if the service failed during an init
 * operation, it stays in the created state, even after stopping it
 * @throws Throwable if necessary
 */

@Test
public void testStopFailedInit() throws Throwable {
  BreakableService svc = new BreakableService(true, false, false);
  assertServiceStateCreated(svc);
  try {
    svc.init(new Configuration());
    fail("Expected a failure, got " + svc);
  } catch (BreakableService.BrokenLifecycleEvent e) {
    //expected
  }
  //the service state wasn't passed
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
  //now try to stop
  svc.stop();
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

@Test
public void testPolicyInitializeAfterSchedulerInitialized() {
  @SuppressWarnings("resource")
  MockRM rm = new MockRM(conf);
  rm.init(conf);
  
  // ProportionalCapacityPreemptionPolicy should be initialized after
  // CapacityScheduler initialized. We will 
  // 1) find SchedulingMonitor from RMActiveService's service list, 
  // 2) check if ResourceCalculator in policy is null or not. 
  // If it's not null, we can come to a conclusion that policy initialized
  // after scheduler got initialized
  for (Service service : rm.getRMActiveService().getServices()) {
    if (service instanceof SchedulingMonitor) {
      ProportionalCapacityPreemptionPolicy policy =
          (ProportionalCapacityPreemptionPolicy) ((SchedulingMonitor) service)
              .getSchedulingEditPolicy();
      assertNotNull(policy.getResourceCalculator());
      return;
    }
  }
  
  fail("Failed to find SchedulingMonitor service, please check what happened");
}
 

public void testDeletionofStagingOnKillLastTry() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
  appMaster.init(conf);
  assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  assertTrue("MRAppMaster isn't stopped",
             appMaster.isInState(Service.STATE.STOPPED));
  verify(fs).delete(stagingJobPath, true);
}
 

/**
 * Test that the {@link BreakableStateChangeListener} is picking up
 * the state changes and that its last event field is as expected.
 */
@Test
public void testEventHistory() {
  register();
  BreakableService service = new BreakableService();
  assertListenerState(listener, Service.STATE.NOTINITED);
  assertEquals(0, listener.getEventCount());
  service.init(new Configuration());
  assertListenerState(listener, Service.STATE.INITED);
  assertSame(service, listener.getLastService());
  assertListenerEventCount(listener, 1);

  service.start();
  assertListenerState(listener, Service.STATE.STARTED);
  assertListenerEventCount(listener, 2);

  service.stop();
  assertListenerState(listener, Service.STATE.STOPPED);
  assertListenerEventCount(listener, 3);
}
 

@Test
public void testServiceFailingNotifications() throws Throwable {
  BreakableService svc = new BreakableService(false, false, false);
  BreakableStateChangeListener listener = new BreakableStateChangeListener();
  listener.setFailingState(Service.STATE.STARTED);
  svc.registerServiceListener(listener);
  svc.init(new Configuration());
  assertEventCount(listener, 1);
  //start this; the listener failed but this won't show
  svc.start();
  //counter went up
  assertEventCount(listener, 2);
  assertEquals(1, listener.getFailureCount());
  //stop the service -this doesn't fail
  svc.stop();
  assertEventCount(listener, 3);
  assertEquals(1, listener.getFailureCount());
  svc.stop();
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

@Test
public void testAuxServiceRecoverySetup() throws IOException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, TEST_DIR.toString());
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      RecoverableServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      RecoverableServiceB.class, Service.class);
  try {
    final AuxServices aux = new AuxServices();
    aux.init(conf);
    Assert.assertEquals(2, aux.getServices().size());
    File auxStorageDir = new File(TEST_DIR,
        AuxServices.STATE_STORE_ROOT_NAME);
    Assert.assertEquals(2, auxStorageDir.listFiles().length);
    aux.close();
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 

@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 

/**
 * Walk the {@link BreakableService} through it's lifecycle, 
 * more to verify that service's counters work than anything else
 * @throws Throwable if necessary
 */
@Test
public void testWalkthrough() throws Throwable {

  BreakableService svc = new BreakableService();
  assertServiceStateCreated(svc);
  assertStateCount(svc, Service.STATE.NOTINITED, 1);
  assertStateCount(svc, Service.STATE.INITED, 0);
  assertStateCount(svc, Service.STATE.STARTED, 0);
  assertStateCount(svc, Service.STATE.STOPPED, 0);
  svc.init(new Configuration());
  assertServiceStateInited(svc);
  assertStateCount(svc, Service.STATE.INITED, 1);
  svc.start();
  assertServiceStateStarted(svc);
  assertStateCount(svc, Service.STATE.STARTED, 1);
  svc.stop();
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

@Override
public void serviceStop() throws Exception {
  try {
    synchronized (serviceMap) {
      for (Service service : serviceMap.values()) {
        if (service.getServiceState() == Service.STATE.STARTED) {
          service.unregisterServiceListener(this);
          service.stop();
        }
      }
      serviceMap.clear();
      serviceMetaData.clear();
    }
  } finally {
    super.serviceStop();
  }
}
 

public void testDeletionofStagingOnKillLastTry() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
  appMaster.init(conf);
  assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  assertTrue("MRAppMaster isn't stopped",
             appMaster.isInState(Service.STATE.STOPPED));
  verify(fs).delete(stagingJobPath, true);
}
 

@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

/**
 * verify that when a service that is not started is stopped, the
 * service enters the stopped state
 * @throws Throwable on a failure
 */
@Test
public void testStopUnstarted() throws Throwable {
  BreakableService svc = new BreakableService();
  svc.stop();
  assertServiceStateStopped(svc);
  assertStateCount(svc, Service.STATE.INITED, 0);
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

@Override
protected void serviceStart() throws Exception {
  if (job.isUber()) {
    MRApps.setupDistributedCacheLocal(getConfig());
    this.containerAllocator = new LocalContainerAllocator(
        this.clientService, this.context, nmHost, nmPort, nmHttpPort
        , containerID);
  } else {
    this.containerAllocator = new RMContainerAllocator(
        this.clientService, this.context);
  }
  ((Service)this.containerAllocator).init(getConfig());
  ((Service)this.containerAllocator).start();
  super.serviceStart();
}
 

@Test
public void testStart() {
  assertEquals(STATE.INITED, webAppProxy.getServiceState());
  webAppProxy.start();
  for (Service service : webAppProxy.getServices()) {
    if (service instanceof WebAppProxy) {
      assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress);
    }
  }
  assertEquals(STATE.STARTED, webAppProxy.getServiceState());
}
 

@Test
public void testStopInInitService() throws Throwable {
  Service service = new StopInInitService();
  BreakableStateChangeListener listener = new BreakableStateChangeListener();
  service.registerServiceListener(listener);
  service.init(new Configuration());
  assertServiceInState(service, Service.STATE.STOPPED);
  assertEventCount(listener, 1);
}
 

/**
 * call init twice
 * @throws Throwable if necessary
 */
@Test
public void testInitTwice() throws Throwable {
  BreakableService svc = new BreakableService();
  Configuration conf = new Configuration();
  conf.set("test.init","t");
  svc.init(conf);
  svc.init(new Configuration());
  assertStateCount(svc, Service.STATE.INITED, 1);
  assertServiceConfigurationContains(svc, "test.init");
}
 

/**
 * Verify that when a service is stopped more than once, no exception
 * is thrown.
 * @throws Throwable if necessary
 */
@Test
public void testStopTwice() throws Throwable {
  BreakableService svc = new BreakableService();
  svc.init(new Configuration());
  svc.start();
  svc.stop();
  assertStateCount(svc, Service.STATE.STOPPED, 1);
  svc.stop();
  assertStateCount(svc, Service.STATE.STOPPED, 1);
}
 

protected void addIfService(Object object, boolean addDispatcher) {
  if (object instanceof Service) {
    Service service = (Service) object;
    ServiceWithDependency sd = new ServiceWithDependency(service);
    services.put(service, sd);
    if(addDispatcher) {
      addIfServiceDependency(service, dispatcher);
    }
  }
}
 

/**
 * Verify that all the events are flushed on stopping the HistoryHandler
 * @throws Exception
 */
@Test
public void testEventsFlushOnStop() throws Exception {

  Configuration conf = new Configuration();
  MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
      .getClass().getName(), true);
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);

  // make sure all events are flushed
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of
   * completed maps
   */
  HistoryContext context = new JobHistory();
  ((JobHistory) context).init(conf);
  Job parsedJob = context.getJob(jobId);
  Assert.assertEquals("CompletedMaps not correct", 1, parsedJob
      .getCompletedMaps());

  Map<TaskId, Task> tasks = parsedJob.getTasks();
  Assert.assertEquals("No of tasks not correct", 1, tasks.size());
  verifyTask(tasks.values().iterator().next());

  Map<TaskId, Task> maps = parsedJob.getTasks(TaskType.MAP);
  Assert.assertEquals("No of maps not correct", 1, maps.size());

  Assert.assertEquals("Job state not currect", JobState.SUCCEEDED,
      parsedJob.getState());
}
 

@Override
protected void serviceStart() throws Exception {
  if (job.isUber()) {
    this.containerLauncher = new LocalContainerLauncher(context,
        (TaskUmbilicalProtocol) taskAttemptListener);
    ((LocalContainerLauncher) this.containerLauncher)
            .setEncryptedSpillKey(encryptedSpillKey);
  } else {
    this.containerLauncher = new ContainerLauncherImpl(context);
  }
  ((Service)this.containerLauncher).init(getConfig());
  ((Service)this.containerLauncher).start();
  super.serviceStart();
}