Java Code Examples for org.apache.hadoop.test.GenericTestUtils#LogCapturer

@Test
public void testCommandStatusReport() {
  GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
      .captureLogs(LOG);

  CommandStatusReportFromDatanode report = this.getStatusReport(Collections
      .emptyList());
  cmdStatusReportHandler.onMessage(report, this);
  assertFalse(logCapturer.getOutput().contains("Delete_Block_Status"));
  assertFalse(logCapturer.getOutput().contains("Replicate_Command_Status"));

  report = this.getStatusReport(this.getCommandStatusList());
  cmdStatusReportHandler.onMessage(report, this);
  assertTrue(logCapturer.getOutput().contains("firing event of type " +
      "Delete_Block_Status"));
  assertTrue(logCapturer.getOutput().contains("type: " +
      "deleteBlocksCommand"));

}
 

@Test
public void testMetaStoreConfigDifferentFromType() throws IOException {

  OzoneConfiguration conf = new OzoneConfiguration();
  conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
  String dbType;
  GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);
  if (storeImpl.equals(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB)) {
    dbType = "RocksDB";
  } else {
    dbType = "LevelDB";
  }

  File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
      + "-" + dbType.toLowerCase() + "-test");
  MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
      .setCreateIfMissing(true).setDbFile(dbDir).setDBType(dbType).build();
  assertTrue(logCapturer.getOutput().contains("Using dbType " + dbType + "" +
      " for metastore"));
  dbStore.close();
  dbStore.destroy();
  FileUtils.deleteDirectory(dbDir);

}
 

@Test
public void testdbTypeNotSet() throws IOException {

  OzoneConfiguration conf = new OzoneConfiguration();
  conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
  GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);

  File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
      + "-" + storeImpl.toLowerCase() + "-test");
  MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
      .setCreateIfMissing(true).setDbFile(dbDir).build();
  assertTrue(logCapturer.getOutput().contains("dbType is null, using dbType" +
      " " + storeImpl));
  dbStore.close();
  dbStore.destroy();
  FileUtils.deleteDirectory(dbDir);

}
 

/**
 * Test that all of {@link OzoneManagerProtocolProtos.Type} enum values are
 * categorized in {@link OmUtils#isReadOnly(OMRequest)}.
 */
@Test
public void testIsReadOnlyCapturesAllCmdTypeEnums() throws Exception {
  GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
      .captureLogs(LoggerFactory.getLogger(OmUtils.class));
  OzoneManagerProtocolProtos.Type[] cmdTypes =
      OzoneManagerProtocolProtos.Type.values();

  for (OzoneManagerProtocolProtos.Type cmdtype : cmdTypes) {
    OMRequest request = OMRequest.newBuilder()
        .setCmdType(cmdtype)
        .setClientId(clientId)
        .build();
    OmUtils.isReadOnly(request);
    assertFalse(cmdtype + " is not categorized in " +
            "OmUtils#isReadyOnly",
        logCapturer.getOutput().contains("CmdType " + cmdtype +" is not " +
            "categorized as readOnly or not."));
    logCapturer.clearOutput();
  }
}
 

@Test
public void testContainerNotFoundWithCommitChunk() throws IOException {
  String testDir =
      GenericTestUtils.getTempPath(TestHddsDispatcher.class.getSimpleName());
  try {
    UUID scmId = UUID.randomUUID();
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(HDDS_DATANODE_DIR_KEY, testDir);
    DatanodeDetails dd = randomDatanodeDetails();
    HddsDispatcher hddsDispatcher = createDispatcher(dd, scmId, conf);
    ContainerCommandRequestProto writeChunkRequest =
        getWriteChunkRequest(dd.getUuidString(), 1L, 1L);

    // send read chunk request and make sure container does not exist
    ContainerCommandResponseProto response =
        hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest), null);
    Assert.assertEquals(
        ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult());
    DispatcherContext dispatcherContext =
        new DispatcherContext.Builder()
            .setContainer2BCSIDMap(Collections.emptyMap())
            .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA)
            .build();

    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
        .captureLogs(HddsDispatcher.LOG);
    // send write chunk request without sending create container
    response = hddsDispatcher.dispatch(writeChunkRequest, dispatcherContext);
    // container should not be found
    Assert.assertEquals(
        ContainerProtos.Result.CONTAINER_NOT_FOUND, response.getResult());

    assertTrue(logCapturer.getOutput().contains(
        "ContainerID " + writeChunkRequest.getContainerID()
            + " does not exist"));
  } finally {
    ContainerMetrics.remove();
    FileUtils.deleteDirectory(new File(testDir));
  }
}
 

@Test
public void testWriteChunkWithCreateContainerFailure() throws IOException {
  String testDir = GenericTestUtils.getTempPath(
      TestHddsDispatcher.class.getSimpleName());
  try {
    UUID scmId = UUID.randomUUID();
    OzoneConfiguration conf = new OzoneConfiguration();
    conf.set(HDDS_DATANODE_DIR_KEY, testDir);
    DatanodeDetails dd = randomDatanodeDetails();
    HddsDispatcher hddsDispatcher = createDispatcher(dd, scmId, conf);
    ContainerCommandRequestProto writeChunkRequest = getWriteChunkRequest(
        dd.getUuidString(), 1L, 1L);

    HddsDispatcher mockDispatcher = Mockito.spy(hddsDispatcher);
    ContainerCommandResponseProto.Builder builder =
        getContainerCommandResponse(writeChunkRequest,
            ContainerProtos.Result.DISK_OUT_OF_SPACE, "");
    // Return DISK_OUT_OF_SPACE response when writing chunk
    // with container creation.
    Mockito.doReturn(builder.build()).when(mockDispatcher)
        .createContainer(writeChunkRequest);

    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
        .captureLogs(HddsDispatcher.LOG);
    // send write chunk request without sending create container
    mockDispatcher.dispatch(writeChunkRequest, null);
    // verify the error log
    assertTrue(logCapturer.getOutput()
        .contains("ContainerID " + writeChunkRequest.getContainerID()
            + " creation failed , Result: DISK_OUT_OF_SPACE"));
  } finally {
    ContainerMetrics.remove();
    FileUtils.deleteDirectory(new File(testDir));
  }
}
 

@Test
public void testOneReplicaPipelineRule() throws Exception {

  // As with 30 nodes, We can create 7 pipelines with replication factor 3.
  // (This is because in node manager for every 10 nodes, 7 nodes are
  // healthy, 2 are stale one is dead.)
  int nodes = 30;
  int pipelineFactorThreeCount = 7;
  int pipelineCountOne = 0;
  setup(nodes, pipelineFactorThreeCount, pipelineCountOne);

  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(
          LoggerFactory.getLogger(SCMSafeModeManager.class));

  List<Pipeline> pipelines = pipelineManager.getPipelines();
  for (int i = 0; i < pipelineFactorThreeCount -1; i++) {
    firePipelineEvent(pipelines.get(i));
  }

  // As 90% of 7 with ceil is 7, if we send 6 pipeline reports, rule
  // validate should be still false.

  GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
      "reported count is 6"), 1000, 5000);

  Assert.assertFalse(rule.validate());

  //Fire last pipeline event from datanode.
  firePipelineEvent(pipelines.get(pipelineFactorThreeCount - 1));

  GenericTestUtils.waitFor(() -> rule.validate(), 1000, 5000);
}
 

@Test
public void testIfCloseContainerEventHadnlerInvoked() {
  GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
      .captureLogs(CloseContainerEventHandler.LOG);
  eventQueue.fireEvent(CLOSE_CONTAINER,
      new ContainerID(Math.abs(RandomUtils.nextInt())));
  eventQueue.processAll(1000);
  Assert.assertTrue(logCapturer.getOutput()
      .contains("Close container Event triggered for container"));
}
 

@Test
public void testCloseContainerEventWithInvalidContainer() {
  long id = Math.abs(RandomUtils.nextInt());
  GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
      .captureLogs(CloseContainerEventHandler.LOG);
  eventQueue.fireEvent(CLOSE_CONTAINER,
      new ContainerID(id));
  eventQueue.processAll(1000);
  Assert.assertTrue(logCapturer.getOutput()
      .contains("Failed to close the container"));
}
 

@Test(timeout = 120000)
public void testLeaderIdUsedOnFirstCall() throws Exception {
  List<Pipeline> pipelines = cluster.getStorageContainerManager()
      .getPipelineManager().getPipelines(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE);
  Assert.assertFalse(pipelines.isEmpty());
  Pipeline ratisPipeline = pipelines.iterator().next();
  Assert.assertTrue(ratisPipeline.isHealthy());
  // Verify correct leader info populated
  GenericTestUtils.waitFor(() -> {
    try {
      return verifyLeaderInfo(ratisPipeline);
    } catch (Exception e) {
      LOG.error("Failed verifying the leader info.", e);
      Assert.fail("Failed verifying the leader info.");
      return false;
    }
  }, 200, 20000);
  // Verify client connects to Leader without NotLeaderException
  XceiverClientRatis xceiverClientRatis =
      XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf);
  Logger.getLogger(GrpcClientProtocolService.class).setLevel(Level.DEBUG);
  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(GrpcClientProtocolService.LOG);
  xceiverClientRatis.connect();
  ContainerProtocolCalls.createContainer(xceiverClientRatis, 1L, null);
  logCapturer.stopCapturing();
  Assert.assertFalse("Client should connect to pipeline leader on first try.",
      logCapturer.getOutput().contains(
          "org.apache.ratis.protocol.NotLeaderException"));
}
 

@Test
public void testOneReplicaPipelineRuleMixedPipelines() throws Exception {

  // As with 30 nodes, We can create 7 pipelines with replication factor 3.
  // (This is because in node manager for every 10 nodes, 7 nodes are
  // healthy, 2 are stale one is dead.)
  int nodes = 30;
  int pipelineCountThree = 7;
  int pipelineCountOne = 21;

  setup(nodes, pipelineCountThree, pipelineCountOne);

  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(
          LoggerFactory.getLogger(SCMSafeModeManager.class));

  List<Pipeline> pipelines =
      pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.ONE);
  for (int i = 0; i < pipelineCountOne; i++) {
    firePipelineEvent(pipelines.get(i));
  }

  GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
      "reported count is 0"), 1000, 5000);

  // fired events for one node ratis pipeline, so we will be still false.
  Assert.assertFalse(rule.validate());

  pipelines =
      pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE);
  for (int i = 0; i < pipelineCountThree - 1; i++) {
    firePipelineEvent(pipelines.get(i));
  }

  GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
      "reported count is 6"), 1000, 5000);

  //Fire last pipeline event from datanode.
  firePipelineEvent(pipelines.get(pipelineCountThree - 1));

  GenericTestUtils.waitFor(() -> rule.validate(), 1000, 5000);
}
 

@Test
public void testHealthyPipelineSafeModeRuleWithMixedPipelines()
    throws Exception {

  String storageDir = GenericTestUtils.getTempPath(
      TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID());

  EventQueue eventQueue = new EventQueue();
  List<ContainerInfo> containers =
          new ArrayList<>(HddsTestUtils.getContainerInfo(1));

  OzoneConfiguration config = new OzoneConfiguration();

  // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are
  // stale and last one is dead, and this repeats. So for a 12 node, 9
  // healthy, 2 stale and one dead.
  MockNodeManager nodeManager = new MockNodeManager(true, 12);
  config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
  // enable pipeline check
  config.setBoolean(
          HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
  config.setBoolean(
          HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);

  SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config);
  try {
    SCMPipelineManager pipelineManager = new SCMPipelineManager(config,
        nodeManager, scmMetadataStore.getPipelineTable(), eventQueue);

    pipelineManager.allowPipelineCreation();
    PipelineProvider mockRatisProvider =
        new MockRatisPipelineProvider(nodeManager,
            pipelineManager.getStateManager(), config, true);
    pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
        mockRatisProvider);

    // Create 3 pipelines
    Pipeline pipeline1 =
        pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
            HddsProtos.ReplicationFactor.ONE);
    Pipeline pipeline2 =
        pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
            HddsProtos.ReplicationFactor.THREE);
    Pipeline pipeline3 =
        pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
            HddsProtos.ReplicationFactor.THREE);


    SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(
        config, containers, pipelineManager, eventQueue);

    HealthyPipelineSafeModeRule healthyPipelineSafeModeRule =
        scmSafeModeManager.getHealthyPipelineSafeModeRule();


    // No pipeline event have sent to SCMSafemodeManager
    Assert.assertFalse(healthyPipelineSafeModeRule.validate());


    GenericTestUtils.LogCapturer logCapturer =
        GenericTestUtils.LogCapturer.captureLogs(LoggerFactory.getLogger(
            SCMSafeModeManager.class));

    // fire event with pipeline create status with ratis type and factor 1
    // pipeline, validate() should return false
    firePipelineEvent(pipeline1, eventQueue);

    GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
        "reported count is 1"),
        1000, 5000);
    Assert.assertFalse(healthyPipelineSafeModeRule.validate());

    firePipelineEvent(pipeline2, eventQueue);
    firePipelineEvent(pipeline3, eventQueue);

    GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(),
        1000, 5000);

  } finally {
    scmMetadataStore.getStore().close();
    FileUtil.fullyDelete(new File(storageDir));
  }

}
 

@Test
public void testCheckVersionResponse() throws Exception {
  OzoneConfiguration conf = SCMTestUtils.getConf();
  conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
      true);
  conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
      true);
  try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
      serverAddress, 1000)) {
    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
        .captureLogs(VersionEndpointTask.LOG);
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    OzoneContainer ozoneContainer = new OzoneContainer(
        datanodeDetails, conf, getContext(datanodeDetails), null);
    rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
    VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
        conf, ozoneContainer);
    EndpointStateMachine.EndPointStates newState = versionTask.call();

    // if version call worked the endpoint should automatically move to the
    // next state.
    Assert.assertEquals(EndpointStateMachine.EndPointStates.REGISTER,
        newState);

    // Now rpcEndpoint should remember the version it got from SCM
    Assert.assertNotNull(rpcEndPoint.getVersion());

    // Now change server scmId, so datanode scmId  will be
    // different from SCM server response scmId
    String newScmId = UUID.randomUUID().toString();
    scmServerImpl.setScmId(newScmId);
    rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
    newState = versionTask.call();
    Assert.assertEquals(EndpointStateMachine.EndPointStates.SHUTDOWN,
          newState);
    List<HddsVolume> volumesList = ozoneContainer.getVolumeSet()
        .getFailedVolumesList();
    Assert.assertTrue(volumesList.size() == 1);
    File expectedScmDir = new File(volumesList.get(0).getHddsRootDir(),
        scmServerImpl.getScmId());
    Assert.assertTrue(logCapturer.getOutput().contains("expected scm " +
        "directory " + expectedScmDir.getAbsolutePath() + " does not " +
        "exist"));
    Assert.assertTrue(ozoneContainer.getVolumeSet().getVolumesList().size()
        == 0);
    Assert.assertTrue(ozoneContainer.getVolumeSet().getFailedVolumesList()
        .size() == 1);

  }
}
 

@Test
public void testDatanodeRegistrationAndReports() throws Exception {
  ReconStorageContainerManagerFacade reconScm =
      (ReconStorageContainerManagerFacade)
      cluster.getReconServer().getReconStorageContainerManager();
  StorageContainerManager scm = cluster.getStorageContainerManager();
  PipelineManager reconPipelineManager = reconScm.getPipelineManager();
  PipelineManager scmPipelineManager = scm.getPipelineManager();

  LambdaTestUtils.await(60000, 5000,
      () -> (reconPipelineManager.getPipelines().size() >= 4));

  // Verify if Recon has all the pipelines from SCM.
  scmPipelineManager.getPipelines().forEach(p -> {
    try {
      assertNotNull(reconPipelineManager.getPipeline(p.getId()));
    } catch (PipelineNotFoundException e) {
      Assert.fail();
    }
  });

  // Verify we can never create a pipeline in Recon.
  LambdaTestUtils.intercept(UnsupportedOperationException.class,
      "Trying to create pipeline in Recon, which is prohibited!",
      () -> reconPipelineManager.createPipeline(RATIS, ONE));

  ContainerManager scmContainerManager = scm.getContainerManager();
  assertTrue(scmContainerManager.getContainerIDs().isEmpty());

  // Verify if all the 3 nodes are registered with Recon.
  NodeManager reconNodeManager = reconScm.getScmNodeManager();
  NodeManager scmNodeManager = scm.getScmNodeManager();
  assertEquals(scmNodeManager.getAllNodes().size(),
      reconNodeManager.getAllNodes().size());

  // Create container
  ContainerManager reconContainerManager = reconScm.getContainerManager();
  ContainerInfo containerInfo =
      scmContainerManager.allocateContainer(RATIS, ONE, "test");
  long containerID = containerInfo.getContainerID();
  Pipeline pipeline =
      scmPipelineManager.getPipeline(containerInfo.getPipelineID());
  XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
  runTestOzoneContainerViaDataNode(containerID, client);

  // Verify Recon picked up the new container that was created.
  assertEquals(scmContainerManager.getContainerIDs(),
      reconContainerManager.getContainerIDs());

  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(ReconNodeManager.LOG);
  reconScm.getEventQueue().fireEvent(CLOSE_CONTAINER,
      containerInfo.containerID());
  GenericTestUtils.waitFor(() -> logCapturer.getOutput()
          .contains("Ignoring unsupported command closeContainerCommand"),
      1000, 20000);
}
 

@Test
public void test2WayCommitForTimeoutException() throws Exception {
  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(XceiverClientRatis.LOG);
  XceiverClientManager clientManager = new XceiverClientManager(conf);

  ContainerWithPipeline container1 = storageContainerLocationClient
      .allocateContainer(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
  XceiverClientSpi xceiverClient = clientManager
      .acquireClient(container1.getPipeline());
  Assert.assertEquals(1, xceiverClient.getRefcount());
  Assert.assertEquals(container1.getPipeline(),
      xceiverClient.getPipeline());
  Pipeline pipeline = xceiverClient.getPipeline();
  TestHelper.createPipelineOnDatanode(pipeline, cluster);
  XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
  XceiverClientReply reply = xceiverClient.sendCommandAsync(
      ContainerTestHelper.getCreateContainerRequest(
          container1.getContainerInfo().getContainerID(),
          xceiverClient.getPipeline()));
  reply.getResponse().get();
  Assert.assertEquals(3, ratisClient.getCommitInfoMap().size());
  for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
    // shutdown the ratis follower
    if (ContainerTestHelper.isRatisFollower(dn, pipeline)) {
      cluster.shutdownHddsDatanode(dn.getDatanodeDetails());
      break;
    }
  }
  reply = xceiverClient.sendCommandAsync(ContainerTestHelper
      .getCloseContainer(pipeline,
          container1.getContainerInfo().getContainerID()));
  reply.getResponse().get();
  xceiverClient.watchForCommit(reply.getLogIndex());

  // commitInfo Map will be reduced to 2 here
  Assert.assertEquals(2, ratisClient.getCommitInfoMap().size());
  clientManager.releaseClient(xceiverClient, false);
  Assert.assertTrue(logCapturer.getOutput().contains("3 way commit failed"));
  Assert.assertTrue(logCapturer.getOutput().contains("TimeoutException"));
  Assert
      .assertTrue(logCapturer.getOutput().contains("Committed by majority"));
  logCapturer.stopCapturing();
}
 

@Test
public void test2WayCommitForRetryfailure() throws Exception {
  OzoneConfiguration conf = new OzoneConfiguration();
  startCluster(conf);
  GenericTestUtils.LogCapturer logCapturer =
      GenericTestUtils.LogCapturer.captureLogs(XceiverClientRatis.LOG);
  XceiverClientManager clientManager = new XceiverClientManager(conf);

  ContainerWithPipeline container1 = storageContainerLocationClient
      .allocateContainer(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
  XceiverClientSpi xceiverClient = clientManager
      .acquireClient(container1.getPipeline());
  Assert.assertEquals(1, xceiverClient.getRefcount());
  Assert.assertEquals(container1.getPipeline(),
      xceiverClient.getPipeline());
  Pipeline pipeline = xceiverClient.getPipeline();
  XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
  XceiverClientReply reply = xceiverClient.sendCommandAsync(
      ContainerTestHelper.getCreateContainerRequest(
          container1.getContainerInfo().getContainerID(),
          xceiverClient.getPipeline()));
  reply.getResponse().get();
  Assert.assertEquals(3, ratisClient.getCommitInfoMap().size());
  // wait for the container to be created on all the nodes
  xceiverClient.watchForCommit(reply.getLogIndex());
  for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
    // shutdown the ratis follower
    if (ContainerTestHelper.isRatisFollower(dn, pipeline)) {
      cluster.shutdownHddsDatanode(dn.getDatanodeDetails());
      break;
    }
  }
  reply = xceiverClient.sendCommandAsync(ContainerTestHelper
      .getCloseContainer(pipeline,
          container1.getContainerInfo().getContainerID()));
  reply.getResponse().get();
  xceiverClient.watchForCommit(reply.getLogIndex());

  // commitInfo Map will be reduced to 2 here
  Assert.assertEquals(2, ratisClient.getCommitInfoMap().size());
  clientManager.releaseClient(xceiverClient, false);
  Assert.assertTrue(logCapturer.getOutput().contains("3 way commit failed"));
  Assert
      .assertTrue(logCapturer.getOutput().contains("Committed by majority"));
  logCapturer.stopCapturing();
  shutdown();
}