org.apache.hadoop.ha.HAServiceProtocol.HAServiceState Java Examples

protected void startRMs(MockRM rm1, Configuration confForRM1, MockRM rm2,
    Configuration confForRM2) throws IOException {
  rm1.init(confForRM1);
  rm1.start();
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm2.init(confForRM2);
  rm2.start();
  Assert.assertTrue(rm2.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm1.adminService.transitionToActive(requestInfo);
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.ACTIVE);
}
 

@Test
public void testFailbackToFaultyServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToActive(anyReqInfo());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());

  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new IOException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
  AlwaysFailFencer.fenceCalled = 0;

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We did not fence svc1 because it cooperated and we didn't force it, 
  // we failed to failover so we fenced svc2, we failed to fence svc2
  // so we did not failback to svc1, ie it's still standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc2, AlwaysFailFencer.fencedSvc);
}
 

protected void startRMs(MockRM rm1, Configuration confForRM1, MockRM rm2,
    Configuration confForRM2) throws IOException {
  rm1.init(confForRM1);
  rm1.start();
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm2.init(confForRM2);
  rm2.start();
  Assert.assertTrue(rm2.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm1.adminService.transitionToActive(requestInfo);
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.ACTIVE);
}
 

synchronized void transitionToStandby(boolean initialize)
    throws Exception {
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.STANDBY) {
    LOG.info("Already in standby state");
    return;
  }

  LOG.info("Transitioning to standby state");
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.ACTIVE) {
    stopActiveServices();
    reinitialize(initialize);
  }
  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
  LOG.info("Transitioned to standby state");
}
 

@Test
public void testFailoverWithoutPermission() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc1.proxy).getServiceStatus();
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc2.proxy).getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover when access is denied");
  } catch (FailoverFailedException ffe) {
    assertTrue(ffe.getCause().getMessage().contains("Access denied"));
  }
}
 

BackupNode startBackupNode(Configuration conf,
                           StartupOption startupOpt,
                           int idx) throws IOException {
  Configuration c = new HdfsConfiguration(conf);
  String dirs = getBackupNodeDir(startupOpt, idx);
  c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
  c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      "${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
      "127.0.0.1:0");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
          "127.0.0.1:0");

  BackupNode bn = (BackupNode)NameNode.createNameNode(
      new String[]{startupOpt.getName()}, c);
  assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
  assertTrue(bn.getRole() + " must be in StandbyState",
             bn.getNamesystem().getHAState()
               .equalsIgnoreCase(HAServiceState.STANDBY.name()));
  return bn;
}
 

/**
 * Set up a mock NN with the bare minimum for a DN to register to it.
 */
private DatanodeProtocolClientSideTranslatorPB setupNNMock(int nnIdx)
    throws Exception {
  DatanodeProtocolClientSideTranslatorPB mock =
      Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class);
  Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0))
      .when(mock).versionRequest();
  
  Mockito.doReturn(DFSTestUtil.getLocalDatanodeRegistration())
    .when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class));
  
  Mockito.doAnswer(new HeartbeatAnswer(nnIdx))
    .when(mock).sendHeartbeat(
        Mockito.any(DatanodeRegistration.class),
        Mockito.any(StorageReport[].class),
        Mockito.anyLong(),
        Mockito.anyLong(),
        Mockito.anyInt(),
        Mockito.anyInt(),
        Mockito.anyInt(),
        Mockito.any(VolumeFailureSummary.class));
  mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0);
  return mock;
}
 

@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

DummyHAService(HAServiceState state, InetSocketAddress address,
    boolean testWithProtoBufRPC) {
  this.state = state;
  this.testWithProtoBufRPC = testWithProtoBufRPC;
  if (testWithProtoBufRPC) {
    this.address = startAndGetRPCServerAddress(address);
  } else {
    this.address = address;
  }
  Configuration conf = new Configuration();
  this.proxy = makeMock(conf, HA_HM_RPC_TIMEOUT_DEFAULT);
  try {
    conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
    this.fencer = Mockito.spy(
        NodeFencer.create(conf, DUMMY_FENCE_KEY));
  } catch (BadFencingConfigurationException e) {
    throw new RuntimeException(e);
  }
  synchronized (instances) {
    instances.add(this);
    this.index = instances.size();
  }
}
 

@Test
public void testFailoverToUnreadyService() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
      .getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to a service that's not ready");
  } catch (FailoverFailedException ffe) {
    // Expected
    if (!ffe.getMessage().contains("injected not ready")) {
      throw ffe;
    }
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);

  // Forcing it means we ignore readyToBecomeActive
  doFailover(svc1, svc2, false, true);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFailoverFromActiveToActive() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to an already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFailoverToUnreadyService() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
      .getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to a service that's not ready");
  } catch (FailoverFailedException ffe) {
    // Expected
    if (!ffe.getMessage().contains("injected not ready")) {
      throw ffe;
    }
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);

  // Forcing it means we ignore readyToBecomeActive
  doFailover(svc1, svc2, false, true);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

BackupNode startBackupNode(Configuration conf,
                           StartupOption startupOpt,
                           int idx) throws IOException {
  Configuration c = new HdfsConfiguration(conf);
  String dirs = getBackupNodeDir(startupOpt, idx);
  c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
  c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      "${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
      "127.0.0.1:0");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
          "127.0.0.1:0");

  BackupNode bn = (BackupNode)NameNode.createNameNode(
      new String[]{startupOpt.getName()}, c);
  assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
  assertTrue(bn.getRole() + " must be in StandbyState",
             bn.getNamesystem().getHAState()
               .equalsIgnoreCase(HAServiceState.STANDBY.name()));
  return bn;
}
 

@Before
public void setupHM() throws InterruptedException, IOException {
  Configuration conf = new Configuration();
  conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
  conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
  
  svc = new DummyHAService(HAServiceState.ACTIVE,
      new InetSocketAddress("0.0.0.0", 0), true);
  hm = new HealthMonitor(conf, svc) {
    @Override
    protected HAServiceProtocol createProxy() throws IOException {
      createProxyCount.incrementAndGet();
      if (throwOOMEOnCreate) {
        throw new OutOfMemoryError("oome");
      }
      return super.createProxy();
    }
  };
  LOG.info("Starting health monitor");
  hm.start();
  
  LOG.info("Waiting for HEALTHY signal");    
  waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY);
}
 

/**
 * Set up two services and their failover controllers. svc1 is started
 * first, so that it enters ACTIVE state, and then svc2 is started,
 * which enters STANDBY
 */
public void start() throws Exception {
  // Format the base dir, should succeed
  thrs = new DummyZKFCThread[2];
  thrs[0] = new DummyZKFCThread(ctx, svcs[0]);
  assertEquals(0, thrs[0].zkfc.run(new String[]{"-formatZK"}));
  ctx.addThread(thrs[0]);
  thrs[0].start();
  
  LOG.info("Waiting for svc0 to enter active state");
  waitForHAState(0, HAServiceState.ACTIVE);
  
  LOG.info("Adding svc1");
  thrs[1] = new DummyZKFCThread(ctx, svcs[1]);
  thrs[1].start();
  waitForHAState(1, HAServiceState.STANDBY);
}
 

public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks) {
  this.conf = conf;
  // Fast check interval so tests run faster
  conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
  svcs = new DummyHAService[2];
  svcs[0] = new DummyHAService(HAServiceState.INITIALIZING,
      new InetSocketAddress("svc1", 1234));
  svcs[0].setSharedResource(sharedResource);
  svcs[1] = new DummyHAService(HAServiceState.INITIALIZING,
      new InetSocketAddress("svc2", 1234));
  svcs[1].setSharedResource(sharedResource);
  
  this.ctx = new TestContext();
  this.zks = zks;
}
 

@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover and did not failback because we fenced
  // svc1 (we forced it), therefore svc1 and svc2 should be standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover and did not failback because we fenced
  // svc1 (we forced it), therefore svc1 and svc2 should be standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
  DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // svc1 went standby then back to active
  verify(svc1.proxy).transitionToStandby(anyReqInfo());
  verify(svc1.proxy).transitionToActive(anyReqInfo());
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverToNonExistantServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
  Mockito.doThrow(new IOException("Failed to connect"))
    .when(svc2).getProxy(Mockito.<Configuration>any(),
        Mockito.anyInt());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to a non-existant standby");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
}
 

@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new HealthCheckFailedException("Failed!"))
      .when(svc2.proxy).monitorHealth();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to unhealthy service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverWithoutPermission() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc1.proxy).getServiceStatus();
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc2.proxy).getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover when access is denied");
  } catch (FailoverFailedException ffe) {
    assertTrue(ffe.getCause().getMessage().contains("Access denied"));
  }
}
 

/**
 * This test case test the {@link BPOfferService#trySendErrorReport} method
 * such that if call to standby namenode times out then that should not 
 * affect the active namenode heartbeat processing since this function 
 * are in writeLock.
 * @throws Exception
 */
@Test
public void testTrySendErrorReportWhenStandbyNNTimesOut() throws Exception {
  BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
  bpos.start();
  try {
    waitForInitialization(bpos);
    // Should start with neither NN as active.
    assertNull(bpos.getActiveNN());
    // Have NN1 claim active at txid 1
    mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
    bpos.triggerHeartbeatForTests();
    // Now mockNN1 is acting like active namenode and mockNN2 as Standby
    assertSame(mockNN1, bpos.getActiveNN());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(0))
        .when(mockNN1).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(1))
        .when(mockNN2).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    String errorString = "Can't send invalid block " + FAKE_BLOCK;
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    Thread.sleep(10000);
    long difference = secondCallTime - firstCallTime;
    assertTrue("Active namenode trySendErrorReport processing "
        + "should be independent of standby namenode trySendErrorReport"
        + " processing ", difference < 5000);
  } finally {
    bpos.stop();
  }
}
 

@Test
public void testFailoverFromStandbyToStandby() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  doFailover(svc1, svc2, false, false);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

/**
 * Checks whether other target node is active or not
 * @param targetNodeToActivate
 * @return true if other target node is active or some other exception 
 * occurred and forceActive was set otherwise false
 * @throws IOException
 */
private boolean isOtherTargetNodeActive(String targetNodeToActivate, boolean forceActive)
    throws IOException  {
  Collection<String> targetIds = getTargetIds(targetNodeToActivate);
  targetIds.remove(targetNodeToActivate);
  for(String targetId : targetIds) {
    HAServiceTarget target = resolveTarget(targetId);
    if (!checkManualStateManagementOK(target)) {
      return true;
    }
    try {
      HAServiceProtocol proto = target.getProxy(getConf(), 5000);
      if(proto.getServiceStatus().getState() == HAServiceState.ACTIVE) {
        errOut.println("transitionToActive: Node " +  targetId +" is already active");
        printUsage(errOut, "-transitionToActive");
        return true;
      }
    } catch (Exception e) {
      //If forceActive switch is false then return true
      if(!forceActive) {
        errOut.println("Unexpected error occurred  " + e.getMessage());
        printUsage(errOut, "-transitionToActive");
        return true; 
      }
    }
  }
  return false;
}
 

/**
 * Test that, when the health monitor indicates bad health status,
 * failover is triggered. Also ensures that graceful active->standby
 * transition is used when possible, falling back to fencing when
 * the graceful approach fails.
 */
@Test(timeout=15000)
public void testAutoFailoverOnBadState() throws Exception {
  try {
    cluster.start();
    DummyHAService svc0 = cluster.getService(0);
    LOG.info("Faking svc0 to change the state, should failover to svc1");
    svc0.state = HAServiceState.STANDBY;
    
    // Should fail back to svc0 at this point
    cluster.waitForHAState(1, HAServiceState.ACTIVE);
  } finally {
    cluster.stop();
  }
}
 

private void doCedeActive(int millisToCede) 
    throws AccessControlException, ServiceFailedException, IOException {
  int timeout = FailoverController.getGracefulFenceTimeout(conf);

  // Lock elector to maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      if (millisToCede <= 0) {
        delayJoiningUntilNanotime = 0;
        recheckElectability();
        return;
      }

      LOG.info("Requested by " + UserGroupInformation.getCurrentUser() +
          " at " + Server.getRemoteAddress() + " to cede active role.");
      boolean needFence = false;
      try {
        localTarget.getProxy(conf, timeout).transitionToStandby(createReqInfo());
        LOG.info("Successfully ensured local node is in standby mode");
      } catch (IOException ioe) {
        LOG.warn("Unable to transition local node to standby: " +
            ioe.getLocalizedMessage());
        LOG.warn("Quitting election but indicating that fencing is " +
            "necessary");
        needFence = true;
      }
      delayJoiningUntilNanotime = System.nanoTime() +
          TimeUnit.MILLISECONDS.toNanos(millisToCede);
      elector.quitElection(needFence);
      serviceState = HAServiceState.INITIALIZING;
    }
  }
  recheckElectability();
}
 

@Override
protected void serviceStop() throws Exception {
  if (webApp != null) {
    webApp.stop();
  }
  if (fetcher != null) {
    fetcher.stop();
  }
  if (configurationProvider != null) {
    configurationProvider.close();
  }
  super.serviceStop();
  transitionToStandby(false);
  rmContext.setHAServiceState(HAServiceState.STOPPING);
}
 

private void doCedeActive(int millisToCede) 
    throws AccessControlException, ServiceFailedException, IOException {
  int timeout = FailoverController.getGracefulFenceTimeout(conf);

  // Lock elector to maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      if (millisToCede <= 0) {
        delayJoiningUntilNanotime = 0;
        recheckElectability();
        return;
      }

      LOG.info("Requested by " + UserGroupInformation.getCurrentUser() +
          " at " + Server.getRemoteAddress() + " to cede active role.");
      boolean needFence = false;
      try {
        localTarget.getProxy(conf, timeout).transitionToStandby(createReqInfo());
        LOG.info("Successfully ensured local node is in standby mode");
      } catch (IOException ioe) {
        LOG.warn("Unable to transition local node to standby: " +
            ioe.getLocalizedMessage());
        LOG.warn("Quitting election but indicating that fencing is " +
            "necessary");
        needFence = true;
      }
      delayJoiningUntilNanotime = System.nanoTime() +
          TimeUnit.MILLISECONDS.toNanos(millisToCede);
      elector.quitElection(needFence);
      serviceState = HAServiceState.INITIALIZING;
    }
  }
  recheckElectability();
}