org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread Java Examples

/**
 * Do a simulated kill all masters and regionservers. Useful when it is
 * impossible to bring the mini-cluster back for clean shutdown.
 */
public void killAll() {
  // Do backups first.
  MasterThread activeMaster = null;
  for (MasterThread masterThread : getMasterThreads()) {
    if (!masterThread.getMaster().isActiveMaster()) {
      masterThread.getMaster().abort("killAll");
    } else {
      activeMaster = masterThread;
    }
  }
  // Do active after.
  if (activeMaster != null) {
    activeMaster.getMaster().abort("killAll");
  }
  for (RegionServerThread rst : getRegionServerThreads()) {
    rst.getRegionServer().abort("killAll");
  }
}
 

/**
 * Starts a region server thread and waits until its processed by master. Throws an exception
 * when it can't start a region server or when the region server is not processed by master
 * within the timeout.
 *
 * @return New RegionServerThread
 */
public JVMClusterUtil.RegionServerThread startRegionServerAndWait(long timeout)
    throws IOException {

  JVMClusterUtil.RegionServerThread t =  startRegionServer();
  ServerName rsServerName = t.getRegionServer().getServerName();

  long start = System.currentTimeMillis();
  ClusterMetrics clusterStatus = getClusterMetrics();
  while ((System.currentTimeMillis() - start) < timeout) {
    if (clusterStatus != null && clusterStatus.getLiveServerMetrics().containsKey(rsServerName)) {
      return t;
    }
    Threads.sleep(100);
  }
  if (t.getRegionServer().isOnline()) {
    throw new IOException("RS: " + rsServerName + " online, but not processed by master");
  } else {
    throw new IOException("RS: " + rsServerName + " is offline");
  }
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers status");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 

@Test
public void testVisibilityLabelsOnWALReplay() throws Exception {
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = createTableAndWriteDataWithLabels(tableName,
      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
        .getRegionServerThreads();
    for (RegionServerThread rsThread : regionServerThreads) {
      rsThread.getRegionServer().abort("Aborting ");
    }
    // Start one new RS
    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
    waitForLabelsRegionAvailability(rs.getRegionServer());
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
  }
}
 

@Test
public void testVisibilityLabelsOnRSRestart() throws Exception {
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
      .getRegionServerThreads();
  for (RegionServerThread rsThread : regionServerThreads) {
    rsThread.getRegionServer().abort("Aborting ");
  }
  // Start one new RS
  RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
  waitForLabelsRegionAvailability(rs.getRegionServer());
  try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
      + ")", PRIVATE)) {
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
  }
}
 

/**
 * Make sure that at least the specified number of region servers
 * are running. We don't count the ones that are currently stopping or are
 * stopped.
 * @param num minimum number of region servers that should be running
 * @return true if we started some servers
 * @throws IOException
 */
public boolean ensureSomeNonStoppedRegionServersAvailable(final int num)
  throws IOException {
  boolean startedServer = ensureSomeRegionServersAvailable(num);

  int nonStoppedServers = 0;
  for (JVMClusterUtil.RegionServerThread rst :
    getMiniHBaseCluster().getRegionServerThreads()) {

    HRegionServer hrs = rst.getRegionServer();
    if (hrs.isStopping() || hrs.isStopped()) {
      LOG.info("A region server is stopped or stopping:"+hrs);
    } else {
      nonStoppedServers++;
    }
  }
  for (int i=nonStoppedServers; i<num; ++i) {
    LOG.info("Started new server=" + getMiniHBaseCluster().startRegionServer());
    startedServer = true;
  }
  return startedServer;
}
 

@Test
public void testKillRS() throws Exception {
  TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.getAdmin().createTable(
    TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
      .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build());
  UTIL.waitTableAvailable(tableName);
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  RegionServerThread thread = UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
    .filter(t -> !t.getRegionServer().getRegions(tableName).isEmpty()).findFirst().get();
  thread.getRegionServer().abort("for testing");
  thread.join();
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 100; i < 200; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  enablePeerAndWaitUntilReplicationDone(200);
  checkOrder(200);
}
 

protected static void rollAllWALs() throws Exception {
  for (RegionServerThread t : UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) {
    t.getRegionServer().getWalRoller().requestRollAll();
  }
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return UTIL.getMiniHBaseCluster()
          .getLiveRegionServerThreads()
          .stream()
          .map(RegionServerThread::getRegionServer)
          .allMatch(HRegionServer::walRollRequestFinished);
    }

    @Override
    public String explainFailure() throws Exception {
      return "Log roll has not finished yet";
    }
  });
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers metrics");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 

@Test
public void test() throws InterruptedException, IOException {
  HMaster master = UTIL.getMiniHBaseCluster().stopMaster(0).getMaster();
  // Shutdown master before shutting down rs
  UTIL.waitFor(30000, () -> !master.isAlive());
  RegionServerThread thread = null;
  for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    if (!t.getRegionServer().getRegions(TABLE_NAME).isEmpty()) {
      thread = t;
      break;
    }
  }
  // shutdown rs
  thread.getRegionServer().abort("For testing");
  thread.join();
  // restart master
  UTIL.getMiniHBaseCluster().startMaster();
  // make sure that we can schedule a SCP for the crashed server which WAL is disabled and bring
  // the region online.
  try (Table table =
    UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(30000).build()) {
    table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(1)));
    assertEquals(1, Bytes.toInt(table.get(new Get(Bytes.toBytes(1))).getValue(CF, CQ)));
  }
}
 

static void clearQuotaCache(HBaseTestingUtility testUtil) {
  for (RegionServerThread rst : testUtil.getMiniHBaseCluster().getRegionServerThreads()) {
    RegionServerRpcQuotaManager quotaManager =
        rst.getRegionServer().getRegionServerRpcQuotaManager();
    QuotaCache quotaCache = quotaManager.getQuotaCache();
    quotaCache.getNamespaceQuotaCache().clear();
    quotaCache.getTableQuotaCache().clear();
    quotaCache.getUserQuotaCache().clear();
    quotaCache.getRegionServerQuotaCache().clear();
  }
}
 

private int getNumberOfOnlineRegions(MiniHBaseCluster cluster) {
  int numFound = 0;
  for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
    numFound += rst.getRegionServer().getNumberOfOnlineRegions();
  }
  for (MasterThread mt : cluster.getMasterThreads()) {
    numFound += mt.getMaster().getNumberOfOnlineRegions();
  }
  return numFound;
}
 

@Test
public void test() throws Exception {
  RegionServerThread rsThread = null;
  for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    if (!t.getRegionServer().getRegions(TABLE_NAME).isEmpty()) {
      rsThread = t;
      break;
    }
  }
  HRegionServer rs = rsThread.getRegionServer();
  RegionInfo hri = rs.getRegions(TABLE_NAME).get(0).getRegionInfo();
  HMaster master = UTIL.getMiniHBaseCluster().getMaster();
  ProcedureExecutor<MasterProcedureEnv> executor = master.getMasterProcedureExecutor();
  DummyRegionProcedure proc = new DummyRegionProcedure(executor.getEnvironment(), hri);
  long procId = master.getMasterProcedureExecutor().submitProcedure(proc);
  proc.waitUntilArrive();
  try (AsyncConnection conn =
    ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get()) {
    AsyncAdmin admin = conn.getAdmin();
    CompletableFuture<Void> future = admin.move(hri.getRegionName());
    rs.abort("For testing!");

    UTIL.waitFor(30000,
      () -> executor.getProcedures().stream()
        .filter(p -> p instanceof TransitRegionStateProcedure)
        .map(p -> (TransitRegionStateProcedure) p)
        .anyMatch(p -> Bytes.equals(hri.getRegionName(), p.getRegion().getRegionName())));
    proc.resume();
    UTIL.waitFor(30000, () -> executor.isFinished(procId));
    // see whether the move region procedure can finish properly
    future.get(30, TimeUnit.SECONDS);
  }
}
 

protected int getRegionServerIndex(ServerName serverName) {
  //we have a small number of region servers, this should be fine for now.
  List<RegionServerThread> servers = getRegionServerThreads();
  for (int i=0; i < servers.size(); i++) {
    if (servers.get(i).getRegionServer().getServerName().equals(serverName)) {
      return i;
    }
  }
  return -1;
}
 

/**
 * Tests the case where killing a primary region with unflushed data recovers
 */
@Test
public void testPrimaryRegionKill() throws Exception {
  try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
      Table table = connection.getTable(htd.getTableName())) {

    HTU.loadNumericRows(table, fam, 0, 1000);

    // wal replication is async, we have to wait until the replication catches up, or we timeout
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);

    // we should not have flushed files now, but data in memstores of primary and secondary
    // kill the primary region replica now, and ensure that when it comes back up, we can still
    // read from it the same data from primary and secondaries
    boolean aborted = false;
    for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
      for (Region r : rs.getRegionServer().getRegions(htd.getTableName())) {
        if (r.getRegionInfo().getReplicaId() == 0) {
          LOG.info("Aborting region server hosting primary region replica");
          rs.getRegionServer().abort("for test");
          aborted = true;
          break;
        }
      }
    }
    assertTrue(aborted);

    // wal replication is async, we have to wait until the replication catches up, or we timeout
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 0, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);
  }

  // restart the region server
  HTU.getMiniHBaseCluster().startRegionServer();
}
 

/**
 * Tests the case where killing a secondary region with unflushed data recovers, and the replica
 * becomes available to read again shortly.
 */
@Test
public void testSecondaryRegionKill() throws Exception {
  try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
      Table table = connection.getTable(htd.getTableName())) {
    HTU.loadNumericRows(table, fam, 0, 1000);

    // wait for some time to ensure that async wal replication does it's magic
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);

    // we should not have flushed files now, but data in memstores of primary and secondary
    // kill the secondary region replica now, and ensure that when it comes back up, we can still
    // read from it the same data
    boolean aborted = false;
    for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
      for (Region r : rs.getRegionServer().getRegions(htd.getTableName())) {
        if (r.getRegionInfo().getReplicaId() == 1) {
          LOG.info("Aborting region server hosting secondary region replica");
          rs.getRegionServer().abort("for test");
          aborted = true;
          break;
        }
      }
    }
    assertTrue(aborted);

    // It takes extra time for replica region is ready for read as during
    // region open process, it needs to ask primary region to do a flush and replica region
    // can open newly flushed hfiles to avoid data out-of-sync.
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    HTU.verifyNumericRows(table, fam, 0, 1000, 2);
  }

  // restart the region server
  HTU.getMiniHBaseCluster().startRegionServer();
}
 

@Test
public void test() throws Exception {
  UTIL.createMultiRegionTable(TABLE_NAME, CF);
  try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
    UTIL.loadTable(table, CF);
  }
  int numRegions = UTIL.getMiniHBaseCluster().getRegions(TABLE_NAME).size();
  RegionServerThread rst0 = UTIL.getMiniHBaseCluster().getRegionServerThreads().get(0);
  RegionServerThread rst1 = UTIL.getMiniHBaseCluster().getRegionServerThreads().get(1);
  HRegionServer liveRS;
  RegionServerThread toKillRSThread;
  if (rst1.getRegionServer().getRegions(TableName.META_TABLE_NAME).isEmpty()) {
    liveRS = rst0.getRegionServer();
    toKillRSThread = rst1;
  } else {
    liveRS = rst1.getRegionServer();
    toKillRSThread = rst0;
  }
  assertTrue(liveRS.getRegions(TABLE_NAME).size() < numRegions);
  UTIL.expireSession(toKillRSThread.getRegionServer().getZooKeeper(), false);
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return liveRS.getRegions(TABLE_NAME).size() == numRegions;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Failover is not finished yet";
    }
  });
  toKillRSThread.getRegionServer().stop("Stop for test");
  // make sure that we can successfully quit
  toKillRSThread.join();
}
 

/**
 * Find regionserver other than the one passed.
 * Can't rely on indexes into list of regionservers since crashed servers
 * occupy an index.
 * @param cluster
 * @param notThisOne
 * @return A regionserver that is not <code>notThisOne</code> or null if none
 * found
 */
private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
    final HRegionServer notThisOne) {
  for (RegionServerThread rst: cluster.getRegionServerThreads()) {
    HRegionServer hrs = rst.getRegionServer();
    if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
    if (hrs.isStopping() || hrs.isStopped()) continue;
    return hrs;
  }
  return null;
}
 

/**
 * @param cluster
 * @param server
 * @param table
 * @return
 */
private List<HRegion> getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server,
    byte[] table) {
  List<HRegion> online = Collections.emptyList();
  for (RegionServerThread rst : cluster.getRegionServerThreads()) {
    // if its the server we are going to kill, get the regions we want to reassign
    if (rst.getRegionServer().getServerName().equals(server)) {
        online = rst.getRegionServer().getRegions(org.apache.hadoop.hbase.TableName.valueOf(table));
        break;
    }
  }
  return online;
}
 

/**
 * @param miniHBaseCluster
 * @param server
 * @param bs
 * @return
 */
private List<HRegion> getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server,
    byte[] table) {
  List<HRegion> online = Collections.emptyList();
  for (RegionServerThread rst : cluster.getRegionServerThreads()) {
    // if its the server we are going to kill, get the regions we want to reassign
    if (rst.getRegionServer().getServerName().equals(server)) {
      online = rst.getRegionServer().getOnlineRegions(table);
      break;
    }
  }
  return online;
}
 

/** Flush the given region in the mini cluster. Since no master, we cannot use HBaseAdmin.flush() */
public static void flushRegion(HBaseTestingUtility HTU, RegionInfo regionInfo)
        throws IOException {
  for (RegionServerThread rst : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
    HRegion region = rst.getRegionServer().getRegionByEncodedName(regionInfo.getEncodedName());
    if (region != null) {
      region.flush(true);
      return;
    }
  }
  throw new IOException("Region to flush cannot be found");
}
 

private NavigableSet<String> getDoubleAssignedRegions(
    MiniHBaseCluster cluster) throws IOException {
  NavigableSet<String> online = new TreeSet<>();
  NavigableSet<String> doubled = new TreeSet<>();
  for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
    for (RegionInfo region : ProtobufUtil.getOnlineRegions(
        rst.getRegionServer().getRSRpcServices())) {
      if(!online.add(region.getRegionNameAsString())) {
        doubled.add(region.getRegionNameAsString());
      }
    }
  }
  return doubled;
}
 

@Test
public void testUserTableClusterScopeQuota() throws Exception {
  final Admin admin = TEST_UTIL.getAdmin();
  final String userName = User.getCurrent().getShortName();
  admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAME, ThrottleType.READ_NUMBER,
    20, TimeUnit.HOURS, QuotaScope.CLUSTER));
  triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME);
  for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
      if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
        int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
        if (rsRegionNum == 0) {
          // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
          break;
        } else if (rsRegionNum == 1) {
          // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
          // Read rows from 1 region, so can read 10 first time and 0 second time
          long count = doGets(20, table);
          assertTrue(count == 0 || count == 10);
        } else if (rsRegionNum == 2) {
          // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
          assertEquals(20, doGets(20, table));
        }
        break;
      }
    }
  }
  admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
  triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME);
}
 

@Test
public void testTableClusterScopeQuota() throws Exception {
  final Admin admin = TEST_UTIL.getAdmin();
  admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAME, ThrottleType.READ_NUMBER, 20,
    TimeUnit.HOURS, QuotaScope.CLUSTER));
  triggerTableCacheRefresh(TEST_UTIL, false, TABLE_NAME);
  for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
      if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
        int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
        if (rsRegionNum == 0) {
          // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
          break;
        } else if (rsRegionNum == 1) {
          // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
          // Read rows from 1 region, so can read 10 first time and 0 second time
          long count = doGets(20, table);
          assertTrue(count == 0 || count == 10);
        } else if (rsRegionNum == 2) {
          // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
          assertEquals(20, doGets(20, table));
        }
        break;
      }
    }
  }
  admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAME));
  triggerTableCacheRefresh(TEST_UTIL, true, TABLE_NAME);
}
 

/**
 * @param cluster
 * @param server
 * @param table
 * @return
 */
private List<HRegion> getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server,
    byte[] table) {
  List<HRegion> online = Collections.emptyList();
  for (RegionServerThread rst : cluster.getRegionServerThreads()) {
    // if its the server we are going to kill, get the regions we want to reassign
    if (rst.getRegionServer().getServerName().equals(server)) {
      online = rst.getRegionServer().getOnlineRegions(org.apache.hadoop.hbase.TableName.valueOf(table));
      break;
    }
  }
  return online;
}
 

public int getNumHFiles(final TableName tableName, final byte[] family) {
  int numHFiles = 0;
  for (RegionServerThread regionServerThread : getMiniHBaseCluster().getRegionServerThreads()) {
    numHFiles+= getNumHFilesForRS(regionServerThread.getRegionServer(), tableName,
                                  family);
  }
  return numHFiles;
}
 

/**
 * Test the whole reconstruction loop. Build a table with regions aaa to zzz and load every one of
 * them multiple times with the same date and do a flush at some point. Kill one of the region
 * servers and scan the table. We should see all the rows.
 */
@Test
public void testReconstruction() throws Exception {
  Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAMILY);

  // Load up the table with simple rows and count them
  int initialCount = TEST_UTIL.loadTable(table, FAMILY);
  int count = TEST_UTIL.countRows(table);

  assertEquals(initialCount, count);

  for (int i = 0; i < 4; i++) {
    TEST_UTIL.loadTable(table, FAMILY);
  }
  RegionServerThread rsThread = TEST_UTIL.getHBaseCluster().getRegionServerThreads().get(0);
  int index = 0;
  LOG.info("Expiring {}", TEST_UTIL.getMiniHBaseCluster().getRegionServer(index));
  TEST_UTIL.expireRegionServerSession(index);
  // make sure that the RS is fully down before reading, so that we will read the data from other
  // RSes.
  TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return !rsThread.isAlive();
    }

    @Override
    public String explainFailure() throws Exception {
      return rsThread.getRegionServer() + " is still alive";
    }
  });
  LOG.info("Starting count");

  int newCount = TEST_UTIL.countRows(table);
  assertEquals(count, newCount);
  table.close();
}
 

protected static void addResourceToRegionServerConfiguration(final HBaseTestingUtility testUtil) {
  // When RegionServer is created in MiniHBaseCluster, it uses HBaseConfiguration.create(conf) of
  // the master Configuration. The create() just copies config params over, it does not do
  // a clone for a historic reason. Properties such as resources are lost during this process.
  // Exposing a new method in HBaseConfiguration causes confusion. Instead, the new hbase-site.xml
  // under test-data directory is added to RegionServer's configuration as a workaround.
  for (RegionServerThread rsThread : testUtil.getMiniHBaseCluster().getRegionServerThreads()) {
    rsThread.getRegionServer().getConfiguration().addResource(
      testUtil.getDataTestDir(SERVER_CONFIG));
  }
}
 

public HRegionServer getOtherRegionServer(HRegionServer rs) {
  for (JVMClusterUtil.RegionServerThread rst :
    getMiniHBaseCluster().getRegionServerThreads()) {
    if (!(rst.getRegionServer() == rs)) {
      return rst.getRegionServer();
    }
  }
  return null;
}
 

private static boolean allReplicasHaveRow(byte[] row) throws IOException {
  for (RegionServerThread t : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    for (HRegion region : t.getRegionServer().getRegions(TABLE_NAME)) {
      if (region.get(new Get(row), false).isEmpty()) {
        return false;
      }
    }
  }
  return true;
}