Java Code Examples for org.apache.hadoop.hbase.Waiter#waitFor()

@Test
public void testReplicationStatusSink() throws Exception {
  try (Admin admin = UTIL2.getConnection().getAdmin()) {
    ServerName server = UTIL2.getHBaseCluster().getRegionServer(0).getServerName();
    ReplicationLoadSink loadSink = getLatestSinkMetric(admin, server);
    //First checks if status of timestamp of last applied op is same as RS start, since no edits
    //were replicated yet
    Assert.assertEquals(loadSink.getTimestampStarted(), loadSink.getTimestampsOfLastAppliedOp());
    //now insert some rows on source, so that it gets delivered to target
    TestReplicationStatus.insertRowsOnSource();
    long wait =
      Waiter.waitFor(UTIL2.getConfiguration(), 10000, (Waiter.Predicate<Exception>) () -> {
        ReplicationLoadSink loadSink1 = getLatestSinkMetric(admin, server);
        return loadSink1.getTimestampsOfLastAppliedOp() > loadSink1.getTimestampStarted();
      });
    Assert.assertNotEquals(-1, wait);
  }
}
 

private void waitForTableToEnterQuotaViolation(TableName tn) throws Exception {
  // Verify that the RegionServer has the quota in violation
  final HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0);
  Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      Map<TableName,SpaceQuotaSnapshot> snapshots =
          rs.getRegionServerSpaceQuotaManager().copyQuotaSnapshots();
      SpaceQuotaSnapshot snapshot = snapshots.get(tn);
      if (snapshot == null) {
        LOG.info("Found no snapshot for " + tn);
        return false;
      }
      LOG.info("Found snapshot " + snapshot);
      return snapshot.getQuotaStatus().isInViolation();
    }
  });
}
 

public static boolean waitForAssignment(AssignmentManager am, RegionInfo regionInfo)
    throws IOException {
  // This method can be called before the regionInfo has made it into the regionStateMap
  // so wait around here a while.
  Waiter.waitFor(am.getConfiguration(), 10000,
    () -> am.getRegionStates().getRegionStateNode(regionInfo) != null);
  RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo);
  // Wait until the region has already been open, or we have a TRSP along with it.
  Waiter.waitFor(am.getConfiguration(), 30000,
    () -> regionNode.isInState(State.OPEN) || regionNode.isInTransition());
  TransitRegionStateProcedure proc = regionNode.getProcedure();
  regionNode.lock();
  try {
    if (regionNode.isInState(State.OPEN)) {
      return true;
    }
    proc = regionNode.getProcedure();
  } finally {
    regionNode.unlock();
  }
  assertNotNull(proc);
  ProcedureSyncWait.waitForProcedureToCompleteIOE(am.getMaster().getMasterProcedureExecutor(),
    proc, 5L * 60 * 1000);
  return true;
}
 

private static void waitPeer(final String peerId,
    ReplicationSourceManager manager, final boolean waitForSource) {
  ReplicationPeers rp = manager.getReplicationPeers();
  Waiter.waitFor(conf, 20000, () -> {
    if (waitForSource) {
      ReplicationSourceInterface rs = manager.getSource(peerId);
      if (rs == null) {
        return false;
      }
      if (rs instanceof ReplicationSourceDummy) {
        return ((ReplicationSourceDummy)rs).isStartup();
      }
      return true;
    } else {
      return (rp.getPeer(peerId) != null);
    }
  });
}
 

@Test
public void testSnapshotHandlers() throws Exception {
  final Configuration conf = HBaseConfiguration.create();
  final Server server = mock(Server.class);
  when(server.getConfiguration()).thenReturn(conf);

  ExecutorService executorService = new ExecutorService("testSnapshotHandlers");
  executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, 1);

  CountDownLatch latch = new CountDownLatch(1);
  CountDownLatch waitForEventToStart = new CountDownLatch(1);
  executorService.submit(new EventHandler(server, EventType.C_M_SNAPSHOT_TABLE) {
    @Override
    public void process() throws IOException {
      waitForEventToStart.countDown();
      try {
        latch.await();
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
      }
    }
  });

  //Wait EventHandler to start
  waitForEventToStart.await(10, TimeUnit.SECONDS);
  int activeCount = executorService.getExecutor(ExecutorType.MASTER_SNAPSHOT_OPERATIONS)
      .getThreadPoolExecutor().getActiveCount();
  Assert.assertEquals(1, activeCount);
  latch.countDown();
  Waiter.waitFor(conf, 3000, () -> {
    int count = executorService.getExecutor(ExecutorType.MASTER_SNAPSHOT_OPERATIONS)
        .getThreadPoolExecutor().getActiveCount();
    return count == 0;
  });
}
 

private void waitForHFilesCountLessorEqual(TableName tn, byte[] cf, int count) throws Exception {
  Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return TEST_UTIL.getNumHFiles(tn, cf) <= count;
    }
  });
}
 

static void waitUntilAllMetaReplicasAreReady(HBaseTestingUtility util,
    ConnectionRegistry registry) {
  Configuration conf = util.getConfiguration();
  int regionReplicaCount = util.getConfiguration().getInt(HConstants.META_REPLICAS_NUM,
      HConstants.DEFAULT_META_REPLICA_NUM);
  Waiter.waitFor(conf, conf.getLong("hbase.client.sync.wait.timeout.msec", 60000), 200, true,
    new ExplainingPredicate<IOException>() {
      @Override
      public String explainFailure() {
        return "Not all meta replicas get assigned";
      }

      @Override
      public boolean evaluate() {
        try {
          RegionLocations locs = registry.getMetaRegionLocations().get();
          if (locs.size() < regionReplicaCount) {
            return false;
          }
          for (int i = 0; i < regionReplicaCount; i++) {
            HRegionLocation loc = locs.getRegionLocation(i);
            // Wait until the replica is served by a region server. There could be delay between
            // the replica being available to the connection and region server opening it.
            Optional<ServerName> rsCarryingReplica =
                getRSCarryingReplica(util, loc.getRegion().getTable(), i);
            if (!rsCarryingReplica.isPresent()) {
              return false;
            }
          }
          return true;
        } catch (Exception e) {
          TestZKConnectionRegistry.LOG.warn("Failed to get meta region locations", e);
          return false;
        }
      }
    });
}
 

/**
 * Test no new Compaction requests are generated after calling stop compactions
 */
@Test
public void testStopStartCompaction() throws IOException {
  // setup a compact/split thread on a mock server
  HRegionServer mockServer = Mockito.mock(HRegionServer.class);
  Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
  final CompactSplit thread = new CompactSplit(mockServer);
  Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
  // setup a region/store with some files
  HStore store = r.getStore(COLUMN_FAMILY);
  createStoreFile(r);
  for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) {
    createStoreFile(r);
  }
  thread.switchCompaction(false);
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
    CompactionLifeCycleTracker.DUMMY, null);
  assertFalse(thread.isCompactionsEnabled());
  int longCompactions = thread.getLongCompactions().getActiveCount();
  int shortCompactions = thread.getShortCompactions().getActiveCount();
  assertEquals("longCompactions=" + longCompactions + "," +
      "shortCompactions=" + shortCompactions, 0, longCompactions + shortCompactions);
  thread.switchCompaction(true);
  assertTrue(thread.isCompactionsEnabled());
  // Make sure no compactions have run.
  assertEquals(0, thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount());
  // Request a compaction and make sure it is submitted successfully.
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
      CompactionLifeCycleTracker.DUMMY, null);
  // Wait until the compaction finishes.
  Waiter.waitFor(UTIL.getConfiguration(), 5000,
      (Waiter.Predicate<Exception>) () -> thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount() == 1);
  // Make sure there are no compactions running.
  assertEquals(0, thread.getLongCompactions().getActiveCount()
      + thread.getShortCompactions().getActiveCount());
}
 

/**
 * Tests that only a single abort is processed when multiple aborts are requested.
 */
@Test
public void testMultiAbort() {
  assertTrue(cluster.getRegionServerThreads().size() > 0);
  JVMClusterUtil.RegionServerThread t = cluster.getRegionServerThreads().get(0);
  assertTrue(t.isAlive());
  HRegionServer rs = t.getRegionServer();
  assertFalse(rs.isAborted());
  RegionServerCoprocessorHost cpHost = rs.getRegionServerCoprocessorHost();
  StopBlockingRegionObserver cp = (StopBlockingRegionObserver)cpHost.findCoprocessor(
      StopBlockingRegionObserver.class.getName());
  // Enable clean abort.
  cp.setStopAllowed(true);
  // Issue two aborts in quick succession.
  // We need a thread pool here, otherwise the abort() runs into SecurityException when running
  // from the fork join pool when setting the context classloader.
  ExecutorService executor = Executors.newFixedThreadPool(2);
  try {
    CompletableFuture.runAsync(() -> rs.abort("Abort 1"), executor);
    CompletableFuture.runAsync(() -> rs.abort("Abort 2"), executor);
    long testTimeoutMs = 10 * 1000;
    Waiter.waitFor(cluster.getConf(), testTimeoutMs, (Waiter.Predicate<Exception>) rs::isStopped);
    // Make sure only one abort is received.
    assertEquals(1, cp.getNumAbortsRequested());
  } finally {
    executor.shutdownNow();
  }
}
 

private void assertTriggerFlushByChanges(int changes) throws InterruptedException {
  int currentFlushCalled = flushCalled.get();
  for (int i = 0; i < changes; i++) {
    flusher.onUpdate();
  }
  Thread.sleep(1000);
  assertEquals(currentFlushCalled, flushCalled.get());
  flusher.onUpdate();
  Waiter.waitFor(conf, 5000, () -> flushCalled.get() == currentFlushCalled + 1);
}
 

@Test
public void testChangingNumberOfPeerRegionServers() throws IOException, InterruptedException {
  LOG.info("testSimplePutDelete");
  MiniHBaseCluster peerCluster = UTIL2.getMiniHBaseCluster();
  // This test wants two RS's up. We only run one generally so add one.
  peerCluster.startRegionServer();
  Waiter.waitFor(peerCluster.getConfiguration(), 30000, new Waiter.Predicate<Exception>() {
    @Override public boolean evaluate() throws Exception {
      return peerCluster.getLiveRegionServerThreads().size() > 1;
    }
  });
  int numRS = peerCluster.getRegionServerThreads().size();

  doPutTest(Bytes.toBytes(1));

  int rsToStop = peerCluster.getServerWithMeta() == 0 ? 1 : 0;
  peerCluster.stopRegionServer(rsToStop);
  peerCluster.waitOnRegionServer(rsToStop);

  // Sanity check
  assertEquals(numRS - 1, peerCluster.getRegionServerThreads().size());

  doPutTest(Bytes.toBytes(2));

  peerCluster.startRegionServer();

  // Sanity check
  assertEquals(numRS, peerCluster.getRegionServerThreads().size());

  doPutTest(Bytes.toBytes(3));
}
 

@Test
public void testWALEntryFilterFromReplicationEndpoint() throws Exception {
  ReplicationPeerConfig rpc =
    new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(CONF1))
      .setReplicationEndpointImpl(ReplicationEndpointWithWALEntryFilter.class.getName());
  // test that we can create mutliple WALFilters reflectively
  rpc.getConfiguration().put(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY,
    EverythingPassesWALEntryFilter.class.getName() + "," +
      EverythingPassesWALEntryFilterSubclass.class.getName());
  hbaseAdmin.addReplicationPeer("testWALEntryFilterFromReplicationEndpoint", rpc);
  // now replicate some data.
  try (Connection connection = ConnectionFactory.createConnection(CONF1)) {
    doPut(connection, Bytes.toBytes("row1"));
    doPut(connection, row);
    doPut(connection, Bytes.toBytes("row2"));
  }

  Waiter.waitFor(CONF1, 60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return ReplicationEndpointForTest.replicateCount.get() >= 1;
    }
  });

  Assert.assertNull(ReplicationEndpointWithWALEntryFilter.ex.get());
  //make sure our reflectively created filter is in the filter chain
  Assert.assertTrue(EverythingPassesWALEntryFilter.hasPassedAnEntry());
  hbaseAdmin.removeReplicationPeer("testWALEntryFilterFromReplicationEndpoint");
}
 

@Test
public void testReplicationEndpointReturnsFalseOnReplicate() throws Exception {
  Assert.assertEquals(0, ReplicationEndpointForTest.replicateCount.get());
  Assert.assertTrue(!ReplicationEndpointReturningFalse.replicated.get());
  int peerCount = hbaseAdmin.listReplicationPeers().size();
  final String id = "testReplicationEndpointReturnsFalseOnReplicate";
  hbaseAdmin.addReplicationPeer(id,
    new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(CONF1))
      .setReplicationEndpointImpl(ReplicationEndpointReturningFalse.class.getName()));
  // This test is flakey and then there is so much stuff flying around in here its, hard to
  // debug.  Peer needs to be up for the edit to make it across. This wait on
  // peer count seems to be a hack that has us not progress till peer is up.
  if (hbaseAdmin.listReplicationPeers().size() <= peerCount) {
    LOG.info("Waiting on peercount to go up from " + peerCount);
    Threads.sleep(100);
  }
  // now replicate some data
  doPut(row);

  Waiter.waitFor(CONF1, 60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      // Looks like replication endpoint returns false unless we put more than 10 edits. We
      // only send over one edit.
      int count = ReplicationEndpointForTest.replicateCount.get();
      LOG.info("count=" + count);
      return ReplicationEndpointReturningFalse.replicated.get();
    }
  });
  if (ReplicationEndpointReturningFalse.ex.get() != null) {
    throw ReplicationEndpointReturningFalse.ex.get();
  }

  hbaseAdmin.removeReplicationPeer("testReplicationEndpointReturnsFalseOnReplicate");
}
 

@Test
public void testCustomReplicationEndpoint() throws Exception {
  // test installing a custom replication endpoint other than the default one.
  hbaseAdmin.addReplicationPeer("testCustomReplicationEndpoint",
      new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(CONF1))
          .setReplicationEndpointImpl(ReplicationEndpointForTest.class.getName()));

  // check whether the class has been constructed and started
  Waiter.waitFor(CONF1, 60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return ReplicationEndpointForTest.contructedCount.get() >= numRegionServers;
    }
  });

  Waiter.waitFor(CONF1, 60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return ReplicationEndpointForTest.startedCount.get() >= numRegionServers;
    }
  });

  Assert.assertEquals(0, ReplicationEndpointForTest.replicateCount.get());

  // now replicate some data.
  doPut(Bytes.toBytes("row42"));

  Waiter.waitFor(CONF1, 60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return ReplicationEndpointForTest.replicateCount.get() >= 1;
    }
  });

  doAssert(Bytes.toBytes("row42"));

  hbaseAdmin.removeReplicationPeer("testCustomReplicationEndpoint");
}
 

/**
 * Tests the replication scenario 0 -> 0. By default
 * {@link BaseReplicationEndpoint#canReplicateToSameCluster()} returns false, so the
 * ReplicationSource should terminate, and no further logs should get enqueued
 */
@Test
public void testLoopedReplication() throws Exception {
  LOG.info("testLoopedReplication");
  startMiniClusters(1);
  createTableOnClusters(table);
  addPeer("1", 0, 0);
  Thread.sleep(SLEEP_TIME);

  // wait for source to terminate
  final ServerName rsName = utilities[0].getHBaseCluster().getRegionServer(0).getServerName();
  Waiter.waitFor(baseConfiguration, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      ClusterMetrics clusterStatus = utilities[0].getAdmin()
          .getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
      ServerMetrics serverLoad = clusterStatus.getLiveServerMetrics().get(rsName);
      List<ReplicationLoadSource> replicationLoadSourceList =
          serverLoad.getReplicationLoadSourceList();
      return replicationLoadSourceList.isEmpty();
    }
  });

  Table[] htables = getHTablesOnClusters(tableName);
  putAndWait(row, famName, htables[0], htables[0]);
  rollWALAndWait(utilities[0], table.getTableName(), row);
  ZKWatcher zkw = utilities[0].getZooKeeperWatcher();
  String queuesZnode = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode,
    ZNodePaths.joinZNode("replication", "rs"));
  List<String> listChildrenNoWatch =
      ZKUtil.listChildrenNoWatch(zkw, ZNodePaths.joinZNode(queuesZnode, rsName.toString()));
  assertEquals(0, listChildrenNoWatch.size());
}
 

private void testHBaseReplicationEndpoint(String tableNameStr, String peerId, boolean isSerial)
    throws IOException {
  TestEndpoint.reset();
  int cellNum = 10000;

  TableName tableName = TableName.valueOf(tableNameStr);
  byte[] family = Bytes.toBytes("f");
  byte[] qualifier = Bytes.toBytes("q");
  TableDescriptor td =
      TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
          .newBuilder(family).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build();
  UTIL.createTable(td, null);

  try (Admin admin = CONN.getAdmin()) {
    ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
        .setClusterKey(getZKClusterKey()).setReplicationEndpointImpl(TestEndpoint.class.getName())
        .setReplicateAllUserTables(false).setSerial(isSerial)
        .setTableCFsMap(ImmutableMap.of(tableName, ImmutableList.of())).build();
    admin.addReplicationPeer(peerId, peerConfig);
  }

  try (Table table = CONN.getTable(tableName)) {
    for (int i = 0; i < cellNum; i++) {
      Put put = new Put(Bytes.toBytes(i)).addColumn(family, qualifier, System.currentTimeMillis(),
        Bytes.toBytes(i));
      table.put(put);
    }
  }
  Waiter.waitFor(CONF, 60000, () -> TestEndpoint.getEntries().size() >= cellNum);

  int index = 0;
  Assert.assertEquals(TestEndpoint.getEntries().size(), cellNum);
  if (!isSerial) {
    Collections.sort(TestEndpoint.getEntries(), (a, b) -> {
      long seqA = a.getKey().getSequenceId();
      long seqB = b.getKey().getSequenceId();
      return seqA == seqB ? 0 : (seqA < seqB ? -1 : 1);
    });
  }
  for (Entry entry : TestEndpoint.getEntries()) {
    Assert.assertEquals(entry.getKey().getTableName(), tableName);
    Assert.assertEquals(entry.getEdit().getCells().size(), 1);
    Cell cell = entry.getEdit().getCells().get(0);
    Assert.assertArrayEquals(
      Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()),
      Bytes.toBytes(index));
    index++;
  }
  Assert.assertEquals(index, cellNum);
}
 

@Test
public void testReplicatorBatching() throws Exception {
  // Clear the tables
  truncateTable(UTIL1, tableName);
  truncateTable(UTIL2, tableName);

  // Replace the peer set up for us by the base class with a wrapper for this test
  hbaseAdmin.addReplicationPeer("testReplicatorBatching",
    new ReplicationPeerConfig().setClusterKey(UTIL2.getClusterKey())
        .setReplicationEndpointImpl(ReplicationEndpointForTest.class.getName()));

  ReplicationEndpointForTest.setBatchCount(0);
  ReplicationEndpointForTest.setEntriesCount(0);
  try {
    ReplicationEndpointForTest.pause();
    try {
      // Queue up a bunch of cells of size 8K. Because of RPC size limits, they will all
      // have to be replicated separately.
      final byte[] valueBytes = new byte[8 * 1024];
      for (int i = 0; i < NUM_ROWS; i++) {
        htable1.put(new Put(Bytes.toBytes("row" + Integer.toString(i))).addColumn(famName, null,
          valueBytes));
      }
    } finally {
      ReplicationEndpointForTest.resume();
    }

    // Wait for replication to complete.
    Waiter.waitFor(CONF1, 60000, new Waiter.ExplainingPredicate<Exception>() {
      @Override
      public boolean evaluate() throws Exception {
        LOG.info("Count=" + ReplicationEndpointForTest.getBatchCount());
        return ReplicationEndpointForTest.getBatchCount() >= NUM_ROWS;
      }

      @Override
      public String explainFailure() throws Exception {
        return "We waited too long for expected replication of " + NUM_ROWS + " entries";
      }
    });

    assertEquals("We sent an incorrect number of batches", NUM_ROWS,
      ReplicationEndpointForTest.getBatchCount());
    assertEquals("We did not replicate enough rows", NUM_ROWS, UTIL2.countRows(htable2));
  } finally {
    hbaseAdmin.removeReplicationPeer("testReplicatorBatching");
  }
}
 

@Test
public void testInterClusterReplication() throws Exception {
  final String id = "testInterClusterReplication";

  List<HRegion> regions = UTIL1.getHBaseCluster().getRegions(tableName);
  int totEdits = 0;

  // Make sure edits are spread across regions because we do region based batching
  // before shipping edits.
  for(HRegion region: regions) {
    RegionInfo hri = region.getRegionInfo();
    byte[] row = hri.getStartKey();
    for (int i = 0; i < 100; i++) {
      if (row.length > 0) {
        Put put = new Put(row);
        put.addColumn(famName, row, row);
        region.put(put);
        totEdits++;
      }
    }
  }

  hbaseAdmin.addReplicationPeer(id,
      new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(CONF2))
          .setReplicationEndpointImpl(InterClusterReplicationEndpointForTest.class.getName()));

  final int numEdits = totEdits;
  Waiter.waitFor(CONF1, 30000, new Waiter.ExplainingPredicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return InterClusterReplicationEndpointForTest.replicateCount.get() == numEdits;
    }

    @Override
    public String explainFailure() throws Exception {
      String failure = "Failed to replicate all edits, expected = " + numEdits
          + " replicated = " + InterClusterReplicationEndpointForTest.replicateCount.get();
      return failure;
    }
  });

  hbaseAdmin.removeReplicationPeer("testInterClusterReplication");
  UTIL1.deleteTableData(tableName);
}
 

/**
 * Tests the non cached version of getRegionLocator by moving a region.
 */
@Test
public void testNonCachedGetRegionLocation() throws Exception {
  // Test Initialization.
  final TableName tableName = name.getTableName();
  byte [] family1 = Bytes.toBytes("f1");
  byte [] family2 = Bytes.toBytes("f2");
  try (Table ignored = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
      Admin admin = TEST_UTIL.getAdmin();
      RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
    List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
    assertEquals(1, allRegionLocations.size());
    RegionInfo regionInfo = allRegionLocations.get(0).getRegion();
    ServerName addrBefore = allRegionLocations.get(0).getServerName();
    // Verify region location before move.
    HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
    HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(),  true);

    assertEquals(addrBefore.getPort(), addrCache.getPort());
    assertEquals(addrBefore.getPort(), addrNoCache.getPort());


    // Make sure more than one server.
    if (TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() <= 1) {
      TEST_UTIL.getMiniHBaseCluster().startRegionServer();
      Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, new Waiter.Predicate<Exception>() {
        @Override public boolean evaluate() throws Exception {
          return TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() > 1;
        }
      });
    }

    ServerName addrAfter = null;
    // Now move the region to a different server.
    for (int i = 0; i < TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
         i++) {
      HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
      ServerName addr = regionServer.getServerName();
      if (addr.getPort() != addrBefore.getPort()) {
        admin.move(regionInfo.getEncodedNameAsBytes(), addr);
        // Wait for the region to move.
        Thread.sleep(5000);
        addrAfter = addr;
        break;
      }
    }

    // Verify the region was moved.
    addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
    addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
    assertNotNull(addrAfter);
    assertTrue(addrAfter.getPort() != addrCache.getPort());
    assertEquals(addrAfter.getPort(), addrNoCache.getPort());
  }
}
 

@Test
public void testQuotaEnforcementsFromRS() throws Exception {
  final long sizeLimit = 1024L * 8L; // 8KB
  final long tableSize = 1024L * 10L; // 10KB
  final int numRegions = 10;
  final TableName tn = helper.createTableWithRegions(numRegions);

  // Define the quota
  QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
      tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
  TEST_UTIL.getAdmin().setQuota(settings);

  // Write at least `tableSize` data
  try {
    helper.writeData(tn, tableSize);
  } catch (RetriesExhaustedWithDetailsException | SpaceLimitingException e) {
    // Pass
  }

  final HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
  final RegionServerSpaceQuotaManager manager = rs.getRegionServerSpaceQuotaManager();
  Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      ActivePolicyEnforcement enforcements = manager.getActiveEnforcements();
      SpaceViolationPolicyEnforcement enforcement = enforcements.getPolicyEnforcement(tn);
      // Signifies that we're waiting on the quota snapshot to be fetched
      if (enforcement instanceof MissingSnapshotViolationPolicyEnforcement) {
        return false;
      }
      return enforcement.getQuotaSnapshot().getQuotaStatus().isInViolation();
    }
  });

  // We obtain the violations for a RegionServer by observing the snapshots
  @SuppressWarnings("unchecked")
  Map<TableName, SpaceQuotaSnapshot> snapshots = (Map<TableName, SpaceQuotaSnapshot>) TEST_UTIL
    .getAdmin().getRegionServerSpaceQuotaSnapshots(rs.getServerName());
  SpaceQuotaSnapshot snapshot = snapshots.get(tn);
  assertNotNull("Did not find snapshot for " + tn, snapshot);
  assertTrue(snapshot.getQuotaStatus().isInViolation());
  assertEquals(SpaceViolationPolicy.NO_INSERTS, snapshot.getQuotaStatus().getPolicy().get());
}