Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegionServer#getRegions()

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  cluster = TEST_UTIL.startMiniCluster(ServerNum);
  table = TEST_UTIL.createTable(tableName, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  TEST_UTIL.waitTableAvailable(tableName, 1000);
  TEST_UTIL.loadTable(table, FAMILY);

  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    for (HRegion region : server.getRegions(tableName)) {
      region.flush(true);
    }
  }

  finder.setConf(TEST_UTIL.getConfiguration());
  finder.setServices(cluster.getMaster());
  finder.setClusterMetrics(cluster.getMaster().getClusterMetrics());
}
 

@Test
public void testInternalGetTopBlockLocation() throws Exception {
  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    for (HRegion region : server.getRegions(tableName)) {
      // get region's hdfs block distribution by region and RegionLocationFinder,
      // they should have same result
      HDFSBlocksDistribution blocksDistribution1 = region.getHDFSBlocksDistribution();
      HDFSBlocksDistribution blocksDistribution2 = finder.getBlockDistribution(region
          .getRegionInfo());
      assertEquals(blocksDistribution1.getUniqueBlocksTotalWeight(),
        blocksDistribution2.getUniqueBlocksTotalWeight());
      if (blocksDistribution1.getUniqueBlocksTotalWeight() != 0) {
        assertEquals(blocksDistribution1.getTopHosts().get(0), blocksDistribution2.getTopHosts()
            .get(0));
      }
    }
  }
}
 

@Test
public void testGetTopBlockLocations() throws Exception {
  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    for (HRegion region : server.getRegions(tableName)) {
      List<ServerName> servers = finder.getTopBlockLocations(region
          .getRegionInfo());
      // test table may have empty region
      if (region.getHDFSBlocksDistribution().getUniqueBlocksTotalWeight() == 0) {
        continue;
      }
      List<String> topHosts = region.getHDFSBlocksDistribution().getTopHosts();
      // rs and datanode may have different host in local machine test
      if (!topHosts.contains(server.getServerName().getHostname())) {
        continue;
      }
      for (int j = 0; j < ServerNum; j++) {
        ServerName serverName = cluster.getRegionServer(j).getServerName();
        assertTrue(servers.contains(serverName));
      }
    }
  }
}
 

@Test
public void testRefreshAndWait() throws Exception {
  finder.getCache().invalidateAll();
  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    List<HRegion> regions = server.getRegions(tableName);
    if (regions.size() <= 0) {
      continue;
    }
    List<RegionInfo> regionInfos = new ArrayList<>(regions.size());
    for (HRegion region : regions) {
      regionInfos.add(region.getRegionInfo());
    }
    finder.refreshAndWait(regionInfos);
    for (RegionInfo regionInfo : regionInfos) {
      assertNotNull(finder.getCache().getIfPresent(regionInfo));
    }
  }
}
 

private HStore getStoreWithName(TableName tableName) {
  MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
  List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
  for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
    HRegionServer hrs = rsts.get(i).getRegionServer();
    for (Region region : hrs.getRegions(tableName)) {
      return ((HRegion) region).getStores().iterator().next();
    }
  }
  return null;
}
 

public List<HRegion> findRegionsForTable(TableName tableName) {
  ArrayList<HRegion> ret = new ArrayList<>();
  for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) {
    HRegionServer hrs = rst.getRegionServer();
    for (Region region : hrs.getRegions(tableName)) {
      if (region.getTableDescriptor().getTableName().equals(tableName)) {
        ret.add((HRegion)region);
      }
    }
  }
  return ret;
}
 

private List<byte[]> getRegionsByServer(int rsId) throws IOException {
  List<byte[]> regionNames = Lists.newArrayList();
  HRegionServer hrs = getRegionServer(rsId);
  for (Region r : hrs.getRegions(TABLE_NAME)) {
    regionNames.add(r.getRegionInfo().getRegionName());
  }
  return regionNames;
}
 

private HStore getStoreWithName(TableName tableName) {
  MiniHBaseCluster cluster = hbtu.getMiniHBaseCluster();
  List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
  for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
    HRegionServer hrs = rsts.get(i).getRegionServer();
    for (Region region : hrs.getRegions(tableName)) {
      return ((HRegion) region).getStores().iterator().next();
    }
  }
  return null;
}
 

public int getNumHFilesForRS(final HRegionServer rs, final TableName tableName,
                             final byte[] family) {
  int numHFiles = 0;
  for (Region region : rs.getRegions(tableName)) {
    numHFiles += region.getStore(family).getStorefilesCount();
  }
  return numHFiles;
}
 

static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName)
        throws IOException {

    for (HRegion r : regionServer.getRegions(tableName)) {
        r.flush(true);
    }

}
 

private HStore getStoreWithName(TableName tableName) {
  MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
  List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
  for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
    HRegionServer hrs = rsts.get(i).getRegionServer();
    for (HRegion region : hrs.getRegions(tableName)) {
      return region.getStores().iterator().next();
    }
  }
  return null;
}
 

@Test
public void test() throws IOException, InterruptedException {
  testUtil.getAdmin().createNamespace(
    NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
  Table table = testUtil.createTable(tableName, families);
  table.put(new Put(Bytes.toBytes("k"))
          .addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
  MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
  List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
  Region region = null;
  for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
    HRegionServer hrs = rsts.get(i).getRegionServer();
    for (Region r : hrs.getRegions(tableName)) {
      region = r;
      break;
    }
  }
  assertNotNull(region);
  Thread.sleep(2000);
  RegionStoreSequenceIds ids =
      testUtil.getHBaseCluster().getMaster()
          .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
  assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
  // This will be the sequenceid just before that of the earliest edit in memstore.
  long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
  assertTrue(storeSequenceId > 0);
  testUtil.getAdmin().flush(tableName);
  Thread.sleep(2000);
  ids =
      testUtil.getHBaseCluster().getMaster()
          .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
  assertTrue(ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
    ids.getLastFlushedSequenceId() > storeSequenceId);
  assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
  table.close();
}
 

public static void waitForTableToBeOnline(final HBaseTestingUtility util,
                                          final TableName tableName)
    throws IOException, InterruptedException {
  HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
  List<HRegion> onlineRegions = rs.getRegions(tableName);
  for (HRegion region : onlineRegions) {
    region.waitForFlushesAndCompactions();
  }
  // Wait up to 60 seconds for a table to be available.
  util.waitFor(60000, util.predicateTableAvailable(tableName));
}
 

/**
 * Test the tuning task of {@link PressureAwareFlushThroughputController}
 */
@Test
public void testFlushThroughputTuning() throws Exception {
  Configuration conf = hbtu.getConfiguration();
  setMaxMinThroughputs(20L * 1024 * 1024, 10L * 1024 * 1024);
  conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DefaultStoreEngine.class.getName());
  conf.setInt(PressureAwareFlushThroughputController.HBASE_HSTORE_FLUSH_THROUGHPUT_TUNE_PERIOD,
    3000);
  hbtu.startMiniCluster(1);
  Connection conn = ConnectionFactory.createConnection(conf);
  hbtu.getAdmin().createTable(TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).setCompactionEnabled(false)
    .build());
  hbtu.waitTableAvailable(tableName);
  HRegionServer regionServer = hbtu.getRSForFirstRegionInTable(tableName);
  double pressure = regionServer.getFlushPressure();
  LOG.debug("Flush pressure before flushing: " + pressure);
  PressureAwareFlushThroughputController throughputController =
      (PressureAwareFlushThroughputController) regionServer.getFlushThroughputController();
  for (HRegion region : regionServer.getRegions()) {
    region.flush(true);
  }
  // We used to assert that the flush pressure is zero but after HBASE-15787 or HBASE-18294 we
  // changed to use heapSize instead of dataSize to calculate the flush pressure, and since
  // heapSize will never be zero, so flush pressure will never be zero either. So we changed the
  // assertion here.
  assertTrue(regionServer.getFlushPressure() < pressure);
  Thread.sleep(5000);
  boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(hbtu.getConfiguration());
  if (tablesOnMaster) {
    // If no tables on the master, this math is off and I'm not sure what it is supposed to be
    // when meta is on the regionserver and not on the master.
    assertEquals(10L * 1024 * 1024, throughputController.getMaxThroughput(), EPSILON);
  }
  Table table = conn.getTable(tableName);
  Random rand = new Random();
  for (int i = 0; i < 10; i++) {
    for (int j = 0; j < 10; j++) {
      byte[] value = new byte[256 * 1024];
      rand.nextBytes(value);
      table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
    }
  }
  Thread.sleep(5000);
  double expectedThroughPut = 10L * 1024 * 1024 * (1 + regionServer.getFlushPressure());
  assertEquals(expectedThroughPut, throughputController.getMaxThroughput(), EPSILON);

  conf.set(FlushThroughputControllerFactory.HBASE_FLUSH_THROUGHPUT_CONTROLLER_KEY,
    NoLimitThroughputController.class.getName());
  regionServer.onConfigurationChange(conf);
  assertTrue(throughputController.isStopped());
  assertTrue(regionServer.getFlushThroughputController() instanceof NoLimitThroughputController);
  conn.close();
}
 

@Test
public void testDecomServerExclusion() throws Exception {
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  HRegionServer excludeServer = cluster.getRegionServer(0);
  List<HRegion> regions = excludeServer.getRegions();
  int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
  TEST_UTIL.getAdmin().decommissionRegionServers(
    Collections.singletonList(excludeServer.getServerName()), false);

  waitForServerDecom(excludeServer);

  HRegionServer sourceRegionServer = cluster.getRegionServer(1);
  String rsName = sourceRegionServer.getServerName().getHostname();
  int port = sourceRegionServer.getServerName().getPort();
  String hostname = rsName + ":" + Integer.toString(port);
  RegionMoverBuilder rmBuilder =
    new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);

  int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
  int sourceServerRegions = sourceRegionServer.getRegions().size();

  try (RegionMover regionMover = rmBuilder.build()) {
    Assert.assertTrue(regionMover.unload());
    LOG.info("Unloading {}", hostname);
    assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
    assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
    LOG.info("Before:" + regionsExcludeServer + " After:" +
      cluster.getRegionServer(1).getNumberOfOnlineRegions());
    List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
    int index = 0;
    for (HRegion hRegion : regionList) {
      Assert.assertEquals(hRegion, regions.get(index++));
    }
    Assert.assertEquals(targetServerRegions + sourceServerRegions,
      cluster.getRegionServer(2).getNumberOfOnlineRegions());
    Assert.assertTrue(regionMover.load());
  }

  TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
    Collections.emptyList());
}
 

@Test
public void testDecomServerExclusionWithAck() throws Exception {
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  HRegionServer excludeServer = cluster.getRegionServer(1);
  List<HRegion> regions = excludeServer.getRegions();
  int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
  TEST_UTIL.getAdmin().decommissionRegionServers(
    Collections.singletonList(excludeServer.getServerName()), false);

  waitForServerDecom(excludeServer);

  HRegionServer regionServer = cluster.getRegionServer(0);
  String rsName = regionServer.getServerName().getHostname();
  int port = regionServer.getServerName().getPort();
  String hostname = rsName + ":" + Integer.toString(port);
  RegionMoverBuilder rmBuilder =
    new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration())
      .ack(true);

  int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
  int sourceServerRegions = regionServer.getRegions().size();

  try (RegionMover regionMover = rmBuilder.build()) {
    Assert.assertTrue(regionMover.unload());
    LOG.info("Unloading {}", hostname);
    assertEquals(0, regionServer.getNumberOfOnlineRegions());
    assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
    LOG.info("Before:" + regionsExcludeServer + " After:" +
      cluster.getRegionServer(1).getNumberOfOnlineRegions());
    List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
    int index = 0;
    for (HRegion hRegion : regionList) {
      Assert.assertEquals(hRegion, regions.get(index++));
    }
    Assert.assertEquals(targetServerRegions + sourceServerRegions,
      cluster.getRegionServer(2).getNumberOfOnlineRegions());
    Assert.assertTrue(regionMover.load());
  }

  TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
    Collections.emptyList());
}
 

/**
 * Test that the store files are archived when a column family is removed.
 * @throws java.io.IOException if there's a problem creating a table.
 * @throws java.lang.InterruptedException problem getting a RegionServer.
 */
@Test
public void testArchiveOnTableFamilyDelete() throws IOException, InterruptedException {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.createTable(tableName, new byte[][] {TEST_FAM, Bytes.toBytes("fam2")});

  List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
  // make sure we only have 1 region serving this table
  assertEquals(1, servingRegions.size());
  HRegion region = servingRegions.get(0);

  // get the parent RS and monitor
  HRegionServer hrs = UTIL.getRSForFirstRegionInTable(tableName);
  FileSystem fs = hrs.getFileSystem();

  // put some data on the region
  LOG.debug("-------Loading table");
  UTIL.loadRegion(region, TEST_FAM);

  // get the hfiles in the region
  List<HRegion> regions = hrs.getRegions(tableName);
  assertEquals("More that 1 region for test table.", 1, regions.size());

  region = regions.get(0);
  // wait for all the compactions to complete
  region.waitForFlushesAndCompactions();

  // disable table to prevent new updates
  UTIL.getAdmin().disableTable(tableName);
  LOG.debug("Disabled table");

  // remove all the files from the archive to get a fair comparison
  clearArchiveDirectory();

  // then get the current store files
  byte[][]columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
  List<String> storeFiles = region.getStoreFileList(columns);

  // then delete the table so the hfiles get archived
  UTIL.getAdmin().deleteColumnFamily(tableName, TEST_FAM);

  assertArchiveFiles(fs, storeFiles, 30000);

  UTIL.deleteTable(tableName);
}
 

@Test
public void testArchiveOnTableDelete() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.createTable(tableName, TEST_FAM);

  List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
  // make sure we only have 1 region serving this table
  assertEquals(1, servingRegions.size());
  HRegion region = servingRegions.get(0);

  // get the parent RS and monitor
  HRegionServer hrs = UTIL.getRSForFirstRegionInTable(tableName);
  FileSystem fs = hrs.getFileSystem();

  // put some data on the region
  LOG.debug("-------Loading table");
  UTIL.loadRegion(region, TEST_FAM);

  // get the hfiles in the region
  List<HRegion> regions = hrs.getRegions(tableName);
  assertEquals("More that 1 region for test table.", 1, regions.size());

  region = regions.get(0);
  // wait for all the compactions to complete
  region.waitForFlushesAndCompactions();

  // disable table to prevent new updates
  UTIL.getAdmin().disableTable(tableName);
  LOG.debug("Disabled table");

  // remove all the files from the archive to get a fair comparison
  clearArchiveDirectory();

  // then get the current store files
  byte[][]columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
  List<String> storeFiles = region.getStoreFileList(columns);

  // then delete the table so the hfiles get archived
  UTIL.deleteTable(tableName);
  LOG.debug("Deleted table");

  assertArchiveFiles(fs, storeFiles, 30000);
}
 

/**
 * Verify all the online region servers has been updated to the
 * latest assignment plan
 * @param plan
 * @throws IOException
 */
private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
  // Verify all region servers contain the correct favored nodes information
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  for (int i = 0; i < SLAVES; i++) {
    HRegionServer rs = cluster.getRegionServer(i);
    for (Region region: rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
      InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
          region.getRegionInfo().getEncodedName());
      String regionName = region.getRegionInfo().getRegionNameAsString();
      List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);

      // All regions are supposed to have favored nodes,
      // except for hbase:meta and ROOT
      if (favoredServerList == null) {
        TableDescriptor desc = region.getTableDescriptor();
        // Verify they are ROOT and hbase:meta regions since no favored nodes
        assertNull(favoredSocketAddress);
        assertTrue("User region " +
            region.getTableDescriptor().getTableName() +
            " should have favored nodes", desc.isMetaRegion());
      } else {
        // For user region, the favored nodes in the region server should be
        // identical to favored nodes in the assignmentPlan
        assertTrue(favoredSocketAddress.length == favoredServerList.size());
        assertTrue(favoredServerList.size() > 0);
        for (int j = 0; j < favoredServerList.size(); j++) {
          InetSocketAddress addrFromRS = favoredSocketAddress[j];
          InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
              favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());

          assertNotNull(addrFromRS);
          assertNotNull(addrFromPlan);
          assertTrue("Region server " + rs.getServerName().getAddress()
              + " has the " + positions[j] +
              " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
              addrFromRS + " which is inconsistent with the plan "
              + addrFromPlan, addrFromRS.equals(addrFromPlan));
        }
      }
    }
  }
}
 

@Test
public void testCoprocessorExec() throws Exception {
  // Set up our ping endpoint service on all regions of our test table
  for (JVMClusterUtil.RegionServerThread thread:
      TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
    HRegionServer rs = thread.getRegionServer();
    for (HRegion region: rs.getRegions(TEST_TABLE)) {
      region.getCoprocessorHost().load(PingCoprocessor.class,
        Coprocessor.PRIORITY_USER, conf);
    }
  }

  // Create users for testing, and grant EXEC privileges on our test table
  // only to user A
  User userA = User.createUserForTesting(conf, "UserA", new String[0]);
  User userB = User.createUserForTesting(conf, "UserB", new String[0]);

  grantOnTable(TEST_UTIL, userA.getShortName(),
    TEST_TABLE, null, null,
    Permission.Action.EXEC);
  try {
    // Create an action for invoking our test endpoint
    AccessTestAction execEndpointAction = new AccessTestAction() {
      @Override
      public Object run() throws Exception {
        try (Connection conn = ConnectionFactory.createConnection(conf);
            Table t = conn.getTable(TEST_TABLE)) {
          BlockingRpcChannel service = t.coprocessorService(HConstants.EMPTY_BYTE_ARRAY);
          PingCoprocessor.newBlockingStub(service).noop(null, NoopRequest.newBuilder().build());
        }
        return null;
      }
    };

    String namespace = TEST_TABLE.getNamespaceAsString();
    // Now grant EXEC to the entire namespace to user B
    grantOnNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC);
    // User B should now be allowed also
    verifyAllowed(execEndpointAction, userA, userB);

    revokeFromNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC);
    // Verify that EXEC permission is checked correctly
    verifyDenied(execEndpointAction, userB);
    verifyAllowed(execEndpointAction, userA);
  } finally {
    // Cleanup, revoke the userA privileges
    revokeFromTable(TEST_UTIL, userA.getShortName(), TEST_TABLE, null, null,
      Permission.Action.EXEC);
  }
}