Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility#getAdmin()

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  String snapshotName = tableName + "-it-" + System.currentTimeMillis();
  Admin admin = util.getAdmin();

  // Don't try the snapshot if we're stopping
  if (context.isStopping()) {
    return;
  }

  getLogger().info("Performing action: Snapshot table {}", tableName);
  admin.snapshot(snapshotName, tableName);
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();
  boolean major = RandomUtils.nextInt(0, 100) < majorRatio;

  getLogger().info("Performing action: Compact table " + tableName + ", major=" + major);
  try {
    if (major) {
      admin.majorCompact(tableName);
    } else {
      admin.compact(tableName);
    }
  } catch (Exception ex) {
    getLogger().warn("Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Before
@Override
public void setUp() throws Exception {
  utility = new HBaseTestingUtility();
  Configuration conf = utility.getConfiguration();
  RSGroupUtil.enableRSGroup(conf);
  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE);
  conf.setInt("hbase.hfile.compaction.discharger.interval", 10);
  utility.startMiniCluster(NUM_SLAVES_BASE);
  MiniHBaseCluster cluster = utility.getHBaseCluster();
  final HMaster master = cluster.getMaster();

  //wait for balancer to come online
  utility.waitFor(60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() {
      return master.isInitialized() &&
          ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
    }
  });
  admin = utility.getAdmin();
}
 

@BeforeClass
public static void setUp() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
  TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1");
  StartMiniClusterOption option =
    StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build();
  TEST_UTIL.startMiniCluster(option);
  cluster = TEST_UTIL.getHBaseCluster();
  master = ((MiniHBaseCluster) cluster).getMaster();
  master.balanceSwitch(false);
  hbaseAdmin = TEST_UTIL.getAdmin();
  // wait till the balancer is in online mode
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return master.isInitialized() &&
        ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() &&
        master.getServerManager().getOnlineServersList().size() >= 3;
    }
  });
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessor.class.getName());
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  Admin admin = util.getAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  Table ht = util.createTable(tableName, new byte[][]{dummy, test});

  Put p = new Put(row1);
  p.addColumn(dummy, dummy, dummy);
  ht.put(p);
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();
  boolean major = RandomUtils.nextInt(0, 100) < majorRatio;

  // Don't try the modify if we're stopping
  if (context.isStopping()) {
    return;
  }

  getLogger().info("Performing action: Compact mob of table " + tableName + ", major=" + major);
  try {
    if (major) {
      admin.majorCompact(tableName, CompactType.MOB);
    } else {
      admin.compact(tableName, CompactType.MOB);
    }
  } catch (Exception ex) {
    getLogger().warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Flush random region of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to flush");
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new RegionInfo[0]));
  getLogger().debug("Flushing region " + region.getRegionNameAsString());
  try {
    admin.flushRegion(region.getRegionName());
  } catch (Exception ex) {
    getLogger().warn("Flush failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.set(ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  // Don't try the flush if we're stopping
  if (context.isStopping()) {
    return;
  }

  getLogger().info("Performing action: Flush table " + tableName);
  try {
    admin.flush(tableName);
  } catch (Exception ex) {
    getLogger().warn("Flush failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Merge random adjacent regions of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.size() < 2) {
    getLogger().info("Table " + tableName + " doesn't have enough regions to merge");
    return;
  }

  int i = RandomUtils.nextInt(0, regions.size() - 1);
  RegionInfo a = regions.get(i++);
  RegionInfo b = regions.get(i);
  getLogger().debug("Merging " + a.getRegionNameAsString() + " and " + b.getRegionNameAsString());

  // Don't try the merge if we're stopping
  if (context.isStopping()) {
    return;
  }

  try {
    admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
  } catch (Exception ex) {
    getLogger().warn("Merge failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Test
public void testMissingRows() throws Exception {
    Connection conn = methodWatcher.createConnection();
    conn.setAutoCommit(false);
    HBaseTestingUtility testingUtility = new HBaseTestingUtility(HConfiguration.unwrapDelegate());
    Admin admin = testingUtility.getAdmin();

    conn.createStatement().executeUpdate("CREATE TABLE A (A1 int, a2 int)");
    conn.createStatement().executeUpdate("INSERT INTO A VALUES (1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1),(1,1)");
    for (int i = 0; i < 10; i++) {
        conn.createStatement().executeUpdate("insert into a select * from a --splice-properties useSpark=false\n");
    }

    String conglomerateNumber = TestUtils.lookupConglomerateNumber(CLASS_NAME, "A", methodWatcher);
    final TableName tableName = TableName.valueOf("splice", conglomerateNumber);
    admin.flush(tableName);

    conn.createStatement().executeUpdate("UPDATE A SET A1 = 2");

    PreparedStatement ps = conn.prepareStatement("SELECT a2 FROM A --splice-properties useSpark=true, splits=1\n");
    try (ResultSet rs = ps.executeQuery()) {
        rs.next();
        int numberOfRows = 1;
        admin.flush(tableName);
        while (rs.next()) {
            numberOfRows++;
            assertNotNull("Failure at row: " + numberOfRows, rs.getObject(1));
            assertEquals(1, rs.getInt(1));
        }
    }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  // Don't try the truncate if we're stopping
  if (context.isStopping()) {
    return;
  }

  boolean preserveSplits = random.nextBoolean();
  getLogger().info("Performing action: Truncate table {} preserve splits {}",
    tableName.getNameAsString(), preserveSplits);
  admin.truncateTable(tableName, preserveSplits);
}
 

public static void createTableAndSnapshot(HBaseTestingUtility util, TableName tableName,
    String snapshotName, int numRegions)
    throws Exception {
  try {
    util.deleteTable(tableName);
  } catch(Exception ex) {
    // ignore
  }

  if (numRegions > 1) {
    util.createTable(tableName, FAMILIES, 1, bbb, yyy, numRegions);
  } else {
    util.createTable(tableName, FAMILIES);
  }
  Admin admin = util.getAdmin();

  // put some stuff in the table
  Table table = util.getConnection().getTable(tableName);
  util.loadTable(table, FAMILIES);

  Path rootDir = CommonFSUtils.getRootDir(util.getConfiguration());
  FileSystem fs = rootDir.getFileSystem(util.getConfiguration());

  SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
      Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);

  // load different values
  byte[] value = Bytes.toBytes("after_snapshot_value");
  util.loadTable(table, FAMILIES, value);

  // cause flush to create new files in the region
  admin.flush(tableName);
  table.close();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  // Set the scanner lease to 20min, so the scanner can't be closed by RegionServer
  TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 1200000);
  TEST_UTIL.getConfiguration()
      .setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100);
  TEST_UTIL.getConfiguration().set("dfs.blocksize", "64000");
  TEST_UTIL.getConfiguration().set("dfs.namenode.fs-limits.min-block-size", "1024");
  TEST_UTIL.getConfiguration().set(TimeToLiveHFileCleaner.TTL_CONF_KEY, "0");
  TEST_UTIL.startMiniCluster(RS_NUMBER);
  admin = TEST_UTIL.getAdmin();
}
 

@BeforeClass
public static void startCluster() throws Exception {
  metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  TEST_UTIL = new HBaseTestingUtility();
  TABLES_ON_MASTER = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
  conf = TEST_UTIL.getConfiguration();
  conf.getLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  // testMobMetrics creates few hfiles and manages compaction manually.
  conf.setInt("hbase.hstore.compactionThreshold", 100);
  conf.setInt("hbase.hstore.compaction.max", 100);
  conf.setInt("hbase.regionserver.periodicmemstoreflusher.rangeofdelayseconds", 4*60);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);

  TEST_UTIL.startMiniCluster();
  cluster = TEST_UTIL.getHBaseCluster();
  cluster.waitForActiveAndReadyMaster();
  admin = TEST_UTIL.getAdmin();
  connection = TEST_UTIL.getConnection();

  while (cluster.getLiveRegionServerThreads().isEmpty() &&
      cluster.getRegionServer(0) == null &&
      rs.getMetrics() == null) {
    Threads.sleep(100);
  }
  rs = cluster.getRegionServer(0);
  metricsRegionServer = rs.getMetrics();
  serverSource = metricsRegionServer.getMetricsSource();
}
 

private void rollWALAndWait(final HBaseTestingUtility utility, final TableName table,
    final byte[] row) throws IOException {
  final Admin admin = utility.getAdmin();
  final MiniHBaseCluster cluster = utility.getMiniHBaseCluster();

  // find the region that corresponds to the given row.
  HRegion region = null;
  for (HRegion candidate : cluster.getRegions(table)) {
    if (HRegion.rowIsInRange(candidate.getRegionInfo(), row)) {
      region = candidate;
      break;
    }
  }
  assertNotNull("Couldn't find the region for row '" + Arrays.toString(row) + "'", region);

  final CountDownLatch latch = new CountDownLatch(1);

  // listen for successful log rolls
  final WALActionsListener listener = new WALActionsListener() {
        @Override
        public void postLogRoll(final Path oldPath, final Path newPath) throws IOException {
          latch.countDown();
        }
      };
  region.getWAL().registerWALActionsListener(listener);

  // request a roll
  admin.rollWALWriter(cluster.getServerHoldingRegion(region.getTableDescriptor().getTableName(),
    region.getRegionInfo().getRegionName()));

  // wait
  try {
    latch.await();
  } catch (InterruptedException exception) {
    LOG.warn("Interrupted while waiting for the wal of '" + region + "' to roll. If later " +
        "replication tests fail, it's probably because we should still be waiting.");
    Thread.currentThread().interrupt();
  }
  region.getWAL().unregisterWALActionsListener(listener);
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();
  boolean major = RandomUtils.nextInt(0, 100) < majorRatio;

  getLogger().info("Performing action: Compact random region of table "
    + tableName + ", major=" + major);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to compact");
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new RegionInfo[0]));

  try {
    if (major) {
      getLogger().debug("Major compacting region " + region.getRegionNameAsString());
      admin.majorCompactRegion(region.getRegionName());
    } else {
      getLogger().debug("Compacting region " + region.getRegionNameAsString());
      admin.compactRegion(region.getRegionName());
    }
  } catch (Exception ex) {
    getLogger().warn("Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Before
public void setup() throws Exception {
  // setup configuration
  conf = HBaseConfiguration.create();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  setVisibilityLabelServiceImpl(conf);
  conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  VisibilityTestUtil.enableVisiblityLabels(conf);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      VisibilityReplication.class.getName());
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      SimpleCP.class.getName());
  // Have to reset conf1 in case zk cluster location different
  // than default
  conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
      ScanLabelGenerator.class);
  conf.set("hbase.superuser", User.getCurrent().getShortName());
  SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
      new String[] { "supergroup" });
  // User.createUserForTesting(conf, User.getCurrent().getShortName(), new
  // String[] { "supergroup" });
  USER1 = User.createUserForTesting(conf, "user1", new String[] {});
  TEST_UTIL = new HBaseTestingUtility(conf);
  TEST_UTIL.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
  zkw1 = new ZKWatcher(conf, "cluster1", null, true);

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf1 = HBaseConfiguration.create(conf);
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessorForTagsAtSink.class.getName());
  // setVisibilityLabelServiceImpl(conf1);
  USER1 = User.createUserForTesting(conf1, "user1", new String[] {});
  TEST_UTIL1 = new HBaseTestingUtility(conf1);
  TEST_UTIL1.setZkCluster(miniZK);
  zkw2 = new ZKWatcher(conf1, "cluster2", null, true);

  TEST_UTIL.startMiniCluster(1);
  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
  TEST_UTIL1.startMiniCluster(1);

  admin = TEST_UTIL.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL1.getClusterKey());
  admin.addReplicationPeer("2", rpc);

  Admin hBaseAdmin = TEST_UTIL.getAdmin();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try {
    hBaseAdmin.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin != null) {
      hBaseAdmin.close();
    }
  }
  Admin hBaseAdmin1 = TEST_UTIL1.getAdmin();
  try {
    hBaseAdmin1.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin1 != null) {
      hBaseAdmin1.close();
    }
  }
  addLabels();
  setAuths(conf);
  setAuths(conf1);
}
 

@Test
public void testForCheckingIfEnableAndDisableWorksFineAfterSwitch()
    throws Exception {
  final int NUM_MASTERS = 2;
  final int NUM_REGIONS_TO_CREATE = 4;

  // Start the cluster
  log("Starting cluster");
  Configuration conf = HBaseConfiguration.create();
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(NUM_MASTERS).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  log("Waiting for active/ready master");
  cluster.waitForActiveAndReadyMaster();

  // Create a table with regions
  final TableName tableName = TableName.valueOf(name.getMethodName());
  byte[] family = Bytes.toBytes("family");
  log("Creating table with " + NUM_REGIONS_TO_CREATE + " regions");
  Table ht = TEST_UTIL.createMultiRegionTable(tableName, family, NUM_REGIONS_TO_CREATE);
  int numRegions = -1;
  try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
    numRegions = r.getStartKeys().length;
  }
  numRegions += 1; // catalogs
  log("Waiting for no more RIT\n");
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);
  log("Disabling table\n");
  TEST_UTIL.getAdmin().disableTable(tableName);

  NavigableSet<String> regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
  assertEquals("The number of regions for the table tableRestart should be 0 and only" +
    "the catalog table should be present.", 1, regions.size());

  List<MasterThread> masterThreads = cluster.getMasterThreads();
  MasterThread activeMaster = null;
  if (masterThreads.get(0).getMaster().isActiveMaster()) {
    activeMaster = masterThreads.get(0);
  } else {
    activeMaster = masterThreads.get(1);
  }
  activeMaster.getMaster().stop(
      "stopping the active master so that the backup can become active");
  cluster.hbaseCluster.waitOnMaster(activeMaster);
  cluster.waitForActiveAndReadyMaster();

  assertTrue("The table should not be in enabled state",
      cluster.getMaster().getTableStateManager().isTableState(
      TableName.valueOf(name.getMethodName()), TableState.State.DISABLED,
      TableState.State.DISABLING));
  log("Enabling table\n");
  // Need a new Admin, the previous one is on the old master
  Admin admin = TEST_UTIL.getAdmin();
  admin.enableTable(tableName);
  admin.close();
  log("Waiting for no more RIT\n");
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);
  log("Verifying there are " + numRegions + " assigned on cluster\n");
  regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
  assertEquals("The assigned regions were not onlined after master" +
    " switch except for the catalog table.", 5, regions.size());
  assertTrue("The table should be in enabled state", cluster.getMaster().getTableStateManager()
    .isTableState(TableName.valueOf(name.getMethodName()), TableState.State.ENABLED));
  ht.close();
  TEST_UTIL.shutdownMiniCluster();
}
 

/**
    * Tests that splitting a region is not blocked indefinitely by UPSERT SELECT load
    */
@Test
   public void testSplitDuringUpsertSelect() throws Exception {
       int numUpsertSelectRunners = 4;
       ExecutorService exec = Executors.newFixedThreadPool(numUpsertSelectRunners);
       try (Connection conn = driver.connect(url, props)) {
           final UpsertSelectRunner upsertSelectRunner =
                   new UpsertSelectRunner(dataTable, 0, 105, 1);
           // keep running slow upsert selects
           SlowBatchRegionObserver.SLOW_MUTATE = true;
           for (int i = 0; i < numUpsertSelectRunners; i++) {
               exec.submit(new UpsertSelectLooper(upsertSelectRunner));
               Thread.sleep(300);
           }

           // keep trying to split the region
           final HBaseTestingUtility utility = getUtility();
           final Admin admin = utility.getAdmin();
           final TableName dataTN = TableName.valueOf(dataTable);
           assertEquals(1, utility.getHBaseCluster().getRegions(dataTN).size());
           utility.waitFor(60000L, 1000, new Waiter.Predicate<Exception>() {
               @Override
               public boolean evaluate() throws Exception {
                   try {
                       List<RegionInfo> regions = admin.getRegions(dataTN);
                       if (regions.size() > 1) {
                           LOGGER.info("Found region was split");
                           return true;
                       }
                       if (regions.size() == 0) {
                           // This happens when region in transition or closed
                           LOGGER.info("No region returned");
                           return false;
                       }
                       ;
                       RegionInfo hRegion = regions.get(0);
                       LOGGER.info("Attempting to split region");
                       admin.splitRegionAsync(hRegion.getRegionName(), Bytes.toBytes(2));
                       return false;
                   } catch (NotServingRegionException | DoNotRetryRegionException re) {
                       // during split
                       return false;
                   } 
               }
           });
       } finally {
           SlowBatchRegionObserver.SLOW_MUTATE = false;
           exec.shutdownNow();
           exec.awaitTermination(60, TimeUnit.SECONDS);
       }
   }