Java Code Examples for org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil#getOnlineRegions()

@Test
public void testShouldUnassignTheRegion() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  createTableWithDefaultConf(tableName);

  RegionInfo info = null;
  HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
  List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
  for (RegionInfo regionInfo : onlineRegions) {
    if (!regionInfo.getTable().isSystemTable()) {
      info = regionInfo;
      ADMIN.unassign(regionInfo.getRegionName(), true);
    }
  }
  boolean isInList = ProtobufUtil.getOnlineRegions(
    rs.getRSRpcServices()).contains(info);
  long timeout = System.currentTimeMillis() + 10000;
  while ((System.currentTimeMillis() < timeout) && (isInList)) {
    Thread.sleep(100);
    isInList = ProtobufUtil.getOnlineRegions(
      rs.getRSRpcServices()).contains(info);
  }

  assertFalse("The region should not be present in online regions list.",
    isInList);
}
 

@Test
public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception {
  final String name = this.name.getMethodName();
  byte[] tableName = Bytes.toBytes(name);
  createTableWithDefaultConf(tableName);

  RegionInfo info = null;
  HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(tableName));
  List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
  for (RegionInfo regionInfo : onlineRegions) {
    if (!regionInfo.isMetaRegion()) {
      if (regionInfo.getRegionNameAsString().contains(name)) {
        info = regionInfo;
        try {
          ADMIN.unassign(Bytes.toBytes("sample"), true);
        } catch (UnknownRegionException nsre) {
          // expected, ignore it
        }
      }
    }
  }
  onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
  assertTrue("The region should be present in online regions list.",
      onlineRegions.contains(info));
}
 

@Test
public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  createTableWithDefaultConf(tableName);

  RegionInfo info = null;
  HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
  List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
  for (RegionInfo regionInfo : onlineRegions) {
    if (!regionInfo.isMetaRegion()) {
      if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) {
        info = regionInfo;
        ADMIN.unassign(regionInfo.getRegionName(), true);
      }
    }
  }

  boolean isInList = ProtobufUtil.getOnlineRegions(
    rs.getRSRpcServices()).contains(info);
  long timeout = System.currentTimeMillis() + 10000;
  while ((System.currentTimeMillis() < timeout) && (isInList)) {
    Thread.sleep(100);
    isInList = ProtobufUtil.getOnlineRegions(
      rs.getRSRpcServices()).contains(info);
  }

  assertFalse("The region should not be present in online regions list.",
    isInList);
}
 

private NavigableSet<String> getDoubleAssignedRegions(
    MiniHBaseCluster cluster) throws IOException {
  NavigableSet<String> online = new TreeSet<>();
  NavigableSet<String> doubled = new TreeSet<>();
  for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
    for (RegionInfo region : ProtobufUtil.getOnlineRegions(
        rst.getRegionServer().getRSRpcServices())) {
      if(!online.add(region.getRegionNameAsString())) {
        doubled.add(region.getRegionNameAsString());
      }
    }
  }
  return doubled;
}
 

private void verifyMethodResult(Class<?> coprocessor, String methodName[], TableName tableName,
    Object value[]) throws IOException {
  try {
    for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
      if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) {
        continue;
      }
      for (RegionInfo r : ProtobufUtil
          .getOnlineRegions(t.getRegionServer().getRSRpcServices())) {
        if (!r.getTable().equals(tableName)) {
          continue;
        }
        RegionCoprocessorHost cph =
            t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost();

        Coprocessor cp = cph.findCoprocessor(coprocessor.getName());
        assertNotNull(cp);
        for (int i = 0; i < methodName.length; ++i) {
          Method m = coprocessor.getMethod(methodName[i]);
          Object o = m.invoke(cp);
          assertTrue("Result of " + coprocessor.getName() + "." + methodName[i]
                  + " is expected to be " + value[i].toString() + ", while we get "
                  + o.toString(), o.equals(value[i]));
        }
      }
    }
  } catch (Exception e) {
    throw new IOException(e.toString());
  }
}
 

private void printOutRegions(final HRegionServer hrs, final String prefix)
    throws IOException {
  List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
  for (RegionInfo region: regions) {
    LOG.info(prefix + region.getRegionNameAsString());
  }
}
 

@Test
public void testRecoveredEdits() throws Exception {
  conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024); // create more than one wal
  startCluster(NUM_RS);

  int numLogLines = 10000;
  SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
  // turn off load balancing to prevent regions from moving around otherwise
  // they will consume recovered.edits
  master.balanceSwitch(false);
  FileSystem fs = master.getMasterFileSystem().getFileSystem();

  List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();

  Path rootdir = CommonFSUtils.getRootDir(conf);

  int numRegions = 50;
  try (Table t = installTable(numRegions)) {
    List<RegionInfo> regions = null;
    HRegionServer hrs = null;
    for (int i = 0; i < NUM_RS; i++) {
      hrs = rsts.get(i).getRegionServer();
      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
      // At least one RS will have >= to average number of regions.
      if (regions.size() >= numRegions / NUM_RS) {
        break;
      }
    }
    Path logDir = new Path(rootdir,
        AbstractFSWALProvider.getWALDirectoryName(hrs.getServerName().toString()));

    LOG.info("#regions = " + regions.size());
    Iterator<RegionInfo> it = regions.iterator();
    while (it.hasNext()) {
      RegionInfo region = it.next();
      if (region.getTable().getNamespaceAsString()
          .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
        it.remove();
      }
    }

    makeWAL(hrs, regions, numLogLines, 100);

    slm.splitLogDistributed(logDir);

    int count = 0;
    for (RegionInfo hri : regions) {
      @SuppressWarnings("deprecation")
      Path editsdir = WALSplitUtil
          .getRegionDirRecoveredEditsDir(CommonFSUtils.getWALRegionDir(conf,
              tableName, hri.getEncodedName()));
      LOG.debug("Checking edits dir " + editsdir);
      FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
        @Override
        public boolean accept(Path p) {
          if (WALSplitUtil.isSequenceIdFile(p)) {
            return false;
          }
          return true;
        }
      });
      LOG.info("Files {}", Arrays.stream(files).map(f -> f.getPath().toString()).
        collect(Collectors.joining(",")));
      assertTrue("Edits dir should have more than a one file", files.length > 1);
      for (int i = 0; i < files.length; i++) {
        int c = countWAL(files[i].getPath(), fs, conf);
        count += c;
      }
      LOG.info(count + " edits in " + files.length + " recovered edits files.");
    }

    // check that the log file is moved
    assertFalse(fs.exists(logDir));
    assertEquals(numLogLines, count);
  }
}
 

@Test
public void testMasterStartsUpWithLogSplittingWork() throws Exception {
  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1);
  startCluster(NUM_RS);

  int numRegionsToCreate = 40;
  int numLogLines = 1000;
  // turn off load balancing to prevent regions from moving around otherwise
  // they will consume recovered.edits
  master.balanceSwitch(false);

  try (Table ht = installTable(numRegionsToCreate)) {
    HRegionServer hrs = findRSToKill(false);
    List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
    makeWAL(hrs, regions, numLogLines, 100);

    // abort master
    abortMaster(cluster);

    // abort RS
    LOG.info("Aborting region server: " + hrs.getServerName());
    hrs.abort("testing");

    // wait for abort completes
    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
      @Override
      public boolean evaluate() throws Exception {
        return cluster.getLiveRegionServerThreads().size() <= NUM_RS - 1;
      }
    });

    Thread.sleep(2000);
    LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size());

    // wait for abort completes
    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
      @Override
      public boolean evaluate() throws Exception {
        return (HBaseTestingUtility.getAllOnlineRegions(cluster)
            .size() >= (numRegionsToCreate + 1));
      }
    });

    LOG.info("Current Open Regions After Master Node Starts Up:" +
        HBaseTestingUtility.getAllOnlineRegions(cluster).size());

    assertEquals(numLogLines, TEST_UTIL.countRows(ht));
  }
}
 

/**
 * Find a RS that has regions of a table.
 * @param hasMetaRegion when true, the returned RS has hbase:meta region as well
 */
private HRegionServer findRSToKill(boolean hasMetaRegion) throws Exception {
  List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
  List<RegionInfo> regions = null;
  HRegionServer hrs = null;

  for (RegionServerThread rst : rsts) {
    hrs = rst.getRegionServer();
    while (rst.isAlive() && !hrs.isOnline()) {
      Thread.sleep(100);
    }
    if (!rst.isAlive()) {
      continue;
    }
    boolean isCarryingMeta = false;
    boolean foundTableRegion = false;
    regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
    for (RegionInfo region : regions) {
      if (region.isMetaRegion()) {
        isCarryingMeta = true;
      }
      if (region.getTable() == tableName) {
        foundTableRegion = true;
      }
      if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
        break;
      }
    }
    if (isCarryingMeta && hasMetaRegion) {
      // clients ask for a RS with META
      if (!foundTableRegion) {
        HRegionServer destRS = hrs;
        // the RS doesn't have regions of the specified table so we need move one to this RS
        List<RegionInfo> tableRegions = TEST_UTIL.getAdmin().getRegions(tableName);
        RegionInfo hri = tableRegions.get(0);
        TEST_UTIL.getAdmin().move(hri.getEncodedNameAsBytes(), destRS.getServerName());
        // wait for region move completes
        RegionStates regionStates =
            TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
        TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
          @Override
          public boolean evaluate() throws Exception {
            ServerName sn = regionStates.getRegionServerOfRegion(hri);
            return (sn != null && sn.equals(destRS.getServerName()));
          }
        });
      }
      return hrs;
    } else if (hasMetaRegion || isCarryingMeta) {
      continue;
    }
    if (foundTableRegion) {
      break;
    }
  }

  return hrs;
}
 

/**
 * Determine if regions are balanced. Figure out the total, divide by the
 * number of online servers, then test if each server is +/- 1 of average
 * rounded up.
 */
private void assertRegionsAreBalanced() throws IOException {
  // TODO: Fix this test.  Old balancer used to run with 'slop'.  New
  // balancer does not.
  boolean success = false;
  float slop = (float)UTIL.getConfiguration().getFloat("hbase.regions.slop", 0.1f);
  if (slop <= 0) slop = 1;

  for (int i = 0; i < 5; i++) {
    success = true;
    // make sure all the regions are reassigned before we test balance
    waitForAllRegionsAssigned();

    long regionCount = UTIL.getMiniHBaseCluster().countServedRegions();
    List<HRegionServer> servers = getOnlineRegionServers();
    double avg = (double)regionCount / (double)servers.size();
    int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
    int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
    // Increase the margin a little to accommodate StochasticLoadBalancer
    if (this.balancerName.contains("StochasticLoadBalancer")) {
      avgLoadPlusSlop++;
      avgLoadMinusSlop--;
    }
    LOG.debug("There are " + servers.size() + " servers and " + regionCount
      + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop
      + ", up border: " + avgLoadPlusSlop + "; attempt: " + i);

    for (HRegionServer server : servers) {
      int serverLoad =
        ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
      LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
      if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
          && serverLoad >= avgLoadMinusSlop)) {
        for (RegionInfo hri :
            ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
          if (hri.isMetaRegion()) serverLoad--;
          // LOG.debug(hri.getRegionNameAsString());
        }
        if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) {
          LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
              " actual: " + serverLoad + " slop: " + slop);
          success = false;
          break;
        }
      }
    }

    if (!success) {
      // one or more servers are not balanced. sleep a little to give it a
      // chance to catch up. then, go back to the retry loop.
      try {
        Thread.sleep(10000);
      } catch (InterruptedException e) {}

      UTIL.getHBaseCluster().getMaster().balance();
      continue;
    }

    // if we get here, all servers were balanced, so we should just return.
    return;
  }
  // if we get here, we tried 5 times and never got to short circuit out of
  // the retry loop, so this is a failure.
  fail("After 5 attempts, region assignments were not balanced.");
}