Java Code Examples for org.apache.hadoop.ha.HAServiceProtocol.HAServiceState#ACTIVE

@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
  AlwaysSucceedFencer.fenceCalled = 0;

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover. We did not fence svc1 because it cooperated
  // and we didn't force it, so we failed back to svc1 and fenced svc2.
  // Note svc2 still thinks it's active, that's OK, we fenced it.
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(1, AlwaysSucceedFencer.fenceCalled);
  assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
}
 

@Test
public void testFailoverToUnreadyService() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
      .getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to a service that's not ready");
  } catch (FailoverFailedException ffe) {
    // Expected
    if (!ffe.getMessage().contains("injected not ready")) {
      throw ffe;
    }
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);

  // Forcing it means we ignore readyToBecomeActive
  doFailover(svc1, svc2, false, true);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFailoverFromFaultyServiceFencingFailure() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToStandby(anyReqInfo());

  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over even though fencing failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverFromFaultyServiceSucceeds() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToStandby(anyReqInfo());

  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  AlwaysSucceedFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, false, false);
  } catch (FailoverFailedException ffe) {
    fail("Faulty active prevented failover");
  }

  // svc1 still thinks it's active, that's OK, it was fenced
  assertEquals(1, AlwaysSucceedFencer.fenceCalled);
  assertSame(svc1, AlwaysSucceedFencer.fencedSvc);
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new HealthCheckFailedException("Failed!"))
      .when(svc2.proxy).monitorHealth();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to unhealthy service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverFromActiveToActive() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to an already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFailbackToFaultyServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToActive(anyReqInfo());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());

  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Test
public void testFailoverFromActiveToActive() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to an already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 

@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 

@Override
public HAServiceStatus getServiceStatus() throws IOException {
  checkUnreachable();
  HAServiceStatus ret = new HAServiceStatus(state);
  if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
    ret.setReadyToBecomeActive();
  }
  return ret;
}
 

public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) {
  if (s == null) return null;
  switch (s.getState()) {
  case ACTIVE:
    return new NNHAStatusHeartbeat(HAServiceState.ACTIVE, s.getTxid());
  case STANDBY:
    return new NNHAStatusHeartbeat(HAServiceState.STANDBY, s.getTxid());
  default:
    throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" + s.getState());
  }
}
 

@Override
public void transitionToActive(StateChangeRequestInfo req) throws ServiceFailedException,
    AccessControlException, IOException {
  activeTransitionCount++;
  checkUnreachable();
  if (failToBecomeActive) {
    throw new ServiceFailedException("injected failure");
  }
  if (sharedResource != null) {
    sharedResource.take(DummyHAService.this);
  }
  state = HAServiceState.ACTIVE;
}
 

private static void setHeartbeatResponse(DatanodeCommand[] cmds)
    throws IOException {
  NNHAStatusHeartbeat ha = new NNHAStatusHeartbeat(HAServiceState.ACTIVE,
      fsImage.getLastAppliedOrWrittenTxId());
  HeartbeatResponse response = new HeartbeatResponse(cmds, ha, null);
  doReturn(response).when(spyNN).sendHeartbeat(
      (DatanodeRegistration) any(),
      (StorageReport[]) any(), anyLong(), anyLong(),
      anyInt(), anyInt(), anyInt(), (VolumeFailureSummary) any());
}
 

@Override
public HAServiceStatus getServiceStatus() throws IOException {
  checkUnreachable();
  HAServiceStatus ret = new HAServiceStatus(state);
  if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
    ret.setReadyToBecomeActive();
  }
  return ret;
}
 

/**
 * Checks whether other target node is active or not
 * @param targetNodeToActivate
 * @return true if other target node is active or some other exception 
 * occurred and forceActive was set otherwise false
 * @throws IOException
 */
private boolean isOtherTargetNodeActive(String targetNodeToActivate, boolean forceActive)
    throws IOException  {
  Collection<String> targetIds = getTargetIds(targetNodeToActivate);
  targetIds.remove(targetNodeToActivate);
  for(String targetId : targetIds) {
    HAServiceTarget target = resolveTarget(targetId);
    if (!checkManualStateManagementOK(target)) {
      return true;
    }
    try {
      HAServiceProtocol proto = target.getProxy(getConf(), 5000);
      if(proto.getServiceStatus().getState() == HAServiceState.ACTIVE) {
        errOut.println("transitionToActive: Node " +  targetId +" is already active");
        printUsage(errOut, "-transitionToActive");
        return true;
      }
    } catch (Exception e) {
      //If forceActive switch is false then return true
      if(!forceActive) {
        errOut.println("Unexpected error occurred  " + e.getMessage());
        printUsage(errOut, "-transitionToActive");
        return true; 
      }
    }
  }
  return false;
}
 

public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) {
  if (s == null) return null;
  switch (s.getState()) {
  case ACTIVE:
    return new NNHAStatusHeartbeat(HAServiceState.ACTIVE, s.getTxid());
  case STANDBY:
    return new NNHAStatusHeartbeat(HAServiceState.STANDBY, s.getTxid());
  default:
    throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" + s.getState());
  }
}
 

/**
 * This test case test the {@link BPOfferService#trySendErrorReport} method
 * such that if call to standby namenode times out then that should not 
 * affect the active namenode heartbeat processing since this function 
 * are in writeLock.
 * @throws Exception
 */
@Test
public void testTrySendErrorReportWhenStandbyNNTimesOut() throws Exception {
  BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
  bpos.start();
  try {
    waitForInitialization(bpos);
    // Should start with neither NN as active.
    assertNull(bpos.getActiveNN());
    // Have NN1 claim active at txid 1
    mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
    bpos.triggerHeartbeatForTests();
    // Now mockNN1 is acting like active namenode and mockNN2 as Standby
    assertSame(mockNN1, bpos.getActiveNN());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(0))
        .when(mockNN1).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(1))
        .when(mockNN2).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    String errorString = "Can't send invalid block " + FAKE_BLOCK;
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    Thread.sleep(10000);
    long difference = secondCallTime - firstCallTime;
    assertTrue("Active namenode trySendErrorReport processing "
        + "should be independent of standby namenode trySendErrorReport"
        + " processing ", difference < 5000);
  } finally {
    bpos.stop();
  }
}
 

public ActiveState() {
  super(HAServiceState.ACTIVE);
}
 

/**
  * This test case doesn't add the reportBadBlock request to
  * {@link BPServiceActor#bpThreadEnqueue} when the Standby namenode throws
  * {@link StandbyException}
  * @throws Exception
  */
@Test
 public void testReportBadBlocksWhenNNThrowsStandbyException()
     throws Exception {
   BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
   bpos.start();
   try {
     waitForInitialization(bpos);
     // Should start with neither NN as active.
     assertNull(bpos.getActiveNN());
     // Have NN1 claim active at txid 1
     mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
     bpos.triggerHeartbeatForTests();
     // Now mockNN1 is acting like active namenode and mockNN2 as Standby
     assertSame(mockNN1, bpos.getActiveNN());
     // Return nothing when active Active Namenode calls reportBadBlocks
     Mockito.doNothing().when(mockNN1).reportBadBlocks
         (Mockito.any(LocatedBlock[].class));

     RemoteException re = new RemoteException(StandbyException.class.
         getName(), "Operation category WRITE is not supported in state "
         + "standby", RpcErrorCodeProto.ERROR_APPLICATION);
     // Return StandbyException wrapped in RemoteException when Standby NN
     // calls reportBadBlocks
     Mockito.doThrow(re).when(mockNN2).reportBadBlocks
         (Mockito.any(LocatedBlock[].class));

     bpos.reportBadBlocks(FAKE_BLOCK, mockFSDataset.getVolume(FAKE_BLOCK)
         .getStorageID(), mockFSDataset.getVolume(FAKE_BLOCK)
         .getStorageType());
     // Send heartbeat so that the BpServiceActor can report bad block to
     // namenode
     bpos.triggerHeartbeatForTests();
     Mockito.verify(mockNN2, Mockito.times(1))
     .reportBadBlocks(Mockito.any(LocatedBlock[].class));

     // Trigger another heartbeat, this will send reportBadBlock again if it
     // is present in the queue.
     bpos.triggerHeartbeatForTests();
     Mockito.verify(mockNN2, Mockito.times(1))
         .reportBadBlocks(Mockito.any(LocatedBlock[].class));
   } finally {
     bpos.stop();
   }
 }