Java Code Examples for org.apache.hadoop.ha.HAServiceProtocol.HAServiceState#STANDBY

@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
  AlwaysSucceedFencer.fenceCalled = 0;

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover. We did not fence svc1 because it cooperated
  // and we didn't force it, so we failed back to svc1 and fenced svc2.
  // Note svc2 still thinks it's active, that's OK, we fenced it.
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(1, AlwaysSucceedFencer.fenceCalled);
  assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
}
 

@Test
public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
  DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // svc1 went standby then back to active
  verify(svc1.proxy).transitionToStandby(anyReqInfo());
  verify(svc1.proxy).transitionToActive(anyReqInfo());
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
  AlwaysSucceedFencer.fenceCalled = 0;

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover. We did not fence svc1 because it cooperated
  // and we didn't force it, so we failed back to svc1 and fenced svc2.
  // Note svc2 still thinks it's active, that's OK, we fenced it.
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(1, AlwaysSucceedFencer.fenceCalled);
  assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
}
 

private synchronized void becomeStandby() {
  LOG.info("ZK Election indicated that " + localTarget +
      " should become standby");
  try {
    int timeout = FailoverController.getGracefulFenceTimeout(conf);
    localTarget.getProxy(conf, timeout).transitionToStandby(createReqInfo());
    LOG.info("Successfully transitioned " + localTarget +
        " to standby state");
  } catch (Exception e) {
    LOG.error("Couldn't transition " + localTarget + " to standby state",
        e);
    // TODO handle this. It's a likely case since we probably got fenced
    // at the same time.
  }
  serviceState = HAServiceState.STANDBY;
}
 

@Test
public void testFailbackToFaultyServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToActive(anyReqInfo());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());

  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverAndFailback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  AlwaysSucceedFencer.fenceCalled = 0;
  doFailover(svc1, svc2, false, false);
  assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);

  AlwaysSucceedFencer.fenceCalled = 0;
  doFailover(svc2, svc1, false, false);
  assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailbackToFaultyServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToActive(anyReqInfo());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());

  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverWithoutPermission() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc1.proxy).getServiceStatus();
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc2.proxy).getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover when access is denied");
  } catch (FailoverFailedException ffe) {
    assertTrue(ffe.getCause().getMessage().contains("Access denied"));
  }
}
 

@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new HealthCheckFailedException("Failed!"))
      .when(svc2.proxy).monitorHealth();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to unhealthy service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Override
public HAServiceStatus getServiceStatus() throws IOException {
  checkUnreachable();
  HAServiceStatus ret = new HAServiceStatus(state);
  if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
    ret.setReadyToBecomeActive();
  }
  return ret;
}
 

@Test
public void testFailoverFromNonExistantServiceWithFencer() throws Exception {
  DummyHAService svc1 = spy(new DummyHAService(null, svc1Addr));
  // Getting a proxy to a dead server will throw IOException on call,
  // not on creation of the proxy.
  HAServiceProtocol errorThrowingProxy = Mockito.mock(HAServiceProtocol.class,
      Mockito.withSettings()
        .defaultAnswer(new ThrowsException(
            new IOException("Could not connect to host")))
        .extraInterfaces(Closeable.class));
  Mockito.doNothing().when((Closeable)errorThrowingProxy).close();

  Mockito.doReturn(errorThrowingProxy).when(svc1).getProxy(
      Mockito.<Configuration>any(),
      Mockito.anyInt());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
  } catch (FailoverFailedException ffe) {
    fail("Non-existant active prevented failover");
  }
  // Verify that the proxy created to try to make it go to standby
  // gracefully used the right rpc timeout
  Mockito.verify(svc1).getProxy(
      Mockito.<Configuration>any(),
      Mockito.eq(
        CommonConfigurationKeys.HA_FC_GRACEFUL_FENCE_TIMEOUT_DEFAULT));
      
  // Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Override
public HAServiceStatus getServiceStatus() throws IOException {
  checkUnreachable();
  HAServiceStatus ret = new HAServiceStatus(state);
  if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
    ret.setReadyToBecomeActive();
  }
  return ret;
}
 

/**
 * Test that, when the health monitor indicates bad health status,
 * failover is triggered. Also ensures that graceful active->standby
 * transition is used when possible, falling back to fencing when
 * the graceful approach fails.
 */
@Test(timeout=15000)
public void testAutoFailoverOnBadState() throws Exception {
  try {
    cluster.start();
    DummyHAService svc0 = cluster.getService(0);
    LOG.info("Faking svc0 to change the state, should failover to svc1");
    svc0.state = HAServiceState.STANDBY;
    
    // Should fail back to svc0 at this point
    cluster.waitForHAState(1, HAServiceState.ACTIVE);
  } finally {
    cluster.stop();
  }
}
 

@Test
public void testFailoverFromStandbyToStandby() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  doFailover(svc1, svc2, false, false);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Override
public void transitionToStandby(StateChangeRequestInfo req) throws ServiceFailedException,
    AccessControlException, IOException {
  checkUnreachable();
  if (failToBecomeStandby) {
    throw new ServiceFailedException("injected failure");
  }
  if (sharedResource != null) {
    sharedResource.release(DummyHAService.this);
  }
  state = HAServiceState.STANDBY;
}
 

@Test
public void testAutomaticFailover()
    throws YarnException, InterruptedException, IOException {
  conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);

  cluster.init(conf);
  cluster.start();
  assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
  verifyConnections();

  failover();
  verifyConnections();

  failover();
  verifyConnections();

  // Make the current Active handle an RMFatalEvent,
  // so it transitions to standby.
  ResourceManager rm = cluster.getResourceManager(
      cluster.getActiveRMIndex());
  rm.handleTransitionToStandBy();
  int maxWaitingAttempts = 2000;
  while (maxWaitingAttempts-- > 0 ) {
    if (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY) {
      break;
    }
    Thread.sleep(1);
  }
  Assert.assertFalse("RM didn't transition to Standby ",
      maxWaitingAttempts == 0);
  verifyConnections();
}
 

public StandbyState() {
  super(HAServiceState.STANDBY);
}
 

public void checkIfStandbyRM() {
  standby = (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY);
}