org.apache.hadoop.hbase.util.PairOfSameType Java Examples

@Override
public PairOfSameType<Integer> getRegionCounts() {
  try {
    if (!master.isInitialized()) {
      return new PairOfSameType<>(0, 0);
    }
    Integer onlineRegionCount = 0;
    Integer offlineRegionCount = 0;

    List<TableDescriptor> descriptors = master.listTableDescriptors(null, null,
        null, false);

    for (TableDescriptor htDesc : descriptors) {
      TableName tableName = htDesc.getTableName();
      Map<RegionState.State, List<RegionInfo>> tableRegions =
          master.getAssignmentManager().getRegionStates()
              .getRegionByStateOfTable(tableName);
      onlineRegionCount += tableRegions.get(RegionState.State.OPEN).size();
      offlineRegionCount += tableRegions.get(RegionState.State.OFFLINE).size();
    }
    return new PairOfSameType<>(onlineRegionCount, offlineRegionCount);
  } catch (IOException e) {
    return new PairOfSameType<>(0, 0);
  }
}
 

/**
 * Splits a region
 * @param r Region to split.
 * @return List of region locations
 */
private List<HRegionLocation> splitRegion(final RegionInfo r)
    throws IOException, InterruptedException, ExecutionException {
  List<HRegionLocation> locations = new ArrayList<>();
  // Split this table in two.
  Admin admin = TEST_UTIL.getAdmin();
  Connection connection = TEST_UTIL.getConnection();
  admin.splitRegionAsync(r.getEncodedNameAsBytes()).get();
  admin.close();
  PairOfSameType<RegionInfo> regions = waitOnDaughters(r);
  if (regions != null) {
    try (RegionLocator rl = connection.getRegionLocator(r.getTable())) {
      locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes()));
      locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes()));
    }
    return locations;
  }
  return locations;
}
 

public PairOfSameType<HRegion> stepsAfterPONR(final Server server,
    final RegionServerServices services, PairOfSameType<HRegion> regions)
    throws IOException {
  openDaughters(server, services, regions.getFirst(), regions.getSecond());
  transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
  return regions;
}
 

/**
 * If daughters no longer hold reference to the parents, delete the parent.
 * @param parent RegionInfo of split offlined parent
 * @param rowContent Content of <code>parent</code> row in
 * <code>metaRegionName</code>
 * @return True if we removed <code>parent</code> from meta table and from
 * the filesystem.
 */
boolean cleanParent(final RegionInfo parent, Result rowContent)
throws IOException {
  // Check whether it is a merged region and if it is clean of references.
  if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
    // Wait until clean of merge parent regions first
    return false;
  }
  // Run checks on each daughter split.
  PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
  Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
  Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
  if (hasNoReferences(a) && hasNoReferences(b)) {
    String daughterA = daughters.getFirst() != null?
        daughters.getFirst().getShortNameToLog(): "null";
    String daughterB = daughters.getSecond() != null?
        daughters.getSecond().getShortNameToLog(): "null";
    LOG.debug("Deleting region " + parent.getShortNameToLog() +
      " because daughters -- " + daughterA + ", " + daughterB +
      " -- no longer hold references");
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
    // Remove from in-memory states
    this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
    this.services.getServerManager().removeRegion(parent);
    return true;
  }
  return false;
}
 

private RegionReplicaInfo(final Result result, final HRegionLocation location) {
  this.row = result != null ? result.getRow() : null;
  this.regionInfo = location != null ? location.getRegion() : null;
  this.regionState = (result != null && regionInfo != null)
    ? RegionStateStore.getRegionState(result, regionInfo)
    : null;
  this.serverName = location != null ? location.getServerName() : null;
  this.seqNum = (location != null) ? location.getSeqNum() : HConstants.NO_SEQNUM;
  this.targetServerName = (result != null && regionInfo != null)
    ? MetaTableAccessor.getTargetServerName(result, regionInfo.getReplicaId())
    : null;
  this.mergeRegionInfo = (result != null)
    ? MetaTableAccessor.getMergeRegionsWithName(result.rawCells())
    : null;

  if (result != null) {
    PairOfSameType<RegionInfo> daughterRegions = MetaTableAccessor.getDaughterRegions(result);
    this.splitRegionInfo = new LinkedHashMap<>();
    if (daughterRegions.getFirst() != null) {
      splitRegionInfo.put(HConstants.SPLITA_QUALIFIER_STR, daughterRegions.getFirst());
    }
    if (daughterRegions.getSecond() != null) {
      splitRegionInfo.put(HConstants.SPLITB_QUALIFIER_STR, daughterRegions.getSecond());
    }
  } else {
    this.splitRegionInfo = null;
  }
}
 

private PairOfSameType<RegionInfo> waitOnDaughters(final RegionInfo r)
    throws IOException {
  long start = System.currentTimeMillis();
  PairOfSameType<RegionInfo> pair = null;
  try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
       Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
    Result result = null;
    RegionInfo region = null;
    while ((System.currentTimeMillis() - start) < 60000) {
      result = metaTable.get(new Get(r.getRegionName()));
      if (result == null) {
        break;
      }
      region = CatalogFamilyFormat.getRegionInfo(result);
      if (region.isSplitParent()) {
        LOG.debug(region.toString() + " IS a parent!");
        pair = MetaTableAccessor.getDaughterRegions(result);
        break;
      }
      Threads.sleep(100);
    }

    if (pair.getFirst() == null || pair.getSecond() == null) {
      throw new IOException("Failed to get daughters, for parent region: " + r);
    }
    return pair;
  }
}
 

@Test
public void testWholesomeMerge() throws Exception {
  LOG.info("Starting " + name.getMethodName());
  final TableName tableName = TableName.valueOf(name.getMethodName());

  try {
    // Create table and load data.
    Table table = createTableAndLoadData(MASTER, tableName);
    // Merge 1st and 2nd region
    mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);

    // Merge 2nd and 3th region
    PairOfSameType<RegionInfo> mergedRegions =
      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2, INITIAL_REGION_NUM - 2);

    verifyRowCount(table, ROWSIZE);

    // Randomly choose one of the two merged regions
    RegionInfo hri = RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    AssignmentManager am = cluster.getMaster().getAssignmentManager();
    RegionStates regionStates = am.getRegionStates();

    // We should not be able to assign it again
    am.assign(hri);
    assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));

    // We should not be able to unassign it either
    am.unassign(hri);
    assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));

    table.close();
  } finally {
    TEST_UTIL.deleteTable(tableName);
  }
}
 

private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
    HMaster master, TableName tablename,
    int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
  PairOfSameType<RegionInfo> mergedRegions =
    requestMergeRegion(master, tablename, regionAnum, regionBnum);
  waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
  return mergedRegions;
}
 

private PairOfSameType<RegionInfo> requestMergeRegion(
    HMaster master, TableName tablename,
    int regionAnum, int regionBnum) throws Exception {
  List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
      .getTableRegionsAndLocations(
          TEST_UTIL.getConnection(), tablename);
  RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
  RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
  ADMIN.mergeRegionsAsync(
    regionA.getEncodedNameAsBytes(),
    regionB.getEncodedNameAsBytes(), false);
  return new PairOfSameType<>(regionA, regionB);
}
 

/**
 * Prepare the regions and region files.
 * @param server Hosting server instance.  Can be null when testing (won't try
 * and update in zk if a null server)
 * @param services Used to online/offline regions.
 * @throws IOException If thrown, transaction failed.
 *    Call {@link #rollback(Server, RegionServerServices)}
 * @return Regions created
 */
/* package */PairOfSameType<HRegion> createDaughters(final Server server,
    final RegionServerServices services) throws IOException {
  LOG.info("Starting split of region " + this.parent);
  if ((server != null && server.isStopped()) ||
      (services != null && services.isStopping())) {
    throw new IOException("Server is stopped or stopping");
  }
  assert !this.parent.lock.writeLock().isHeldByCurrentThread():
    "Unsafe to hold write lock while performing RPCs";

  // Coprocessor callback
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplit();
  }

  // Coprocessor callback
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplit(this.splitrow);
  }

  // If true, no cluster to write meta edits to or to update znodes in.
  boolean testing = server == null? true:
      server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
  this.fileSplitTimeout = testing ? this.fileSplitTimeout :
      server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
        this.fileSplitTimeout);

  PairOfSameType<HRegion> daughterRegions = stepsBeforePONR(server, services, testing);

  List<Mutation> metaEntries = new ArrayList<Mutation>();
  if (this.parent.getCoprocessorHost() != null) {
    if (this.parent.getCoprocessorHost().
        preSplitBeforePONR(this.splitrow, metaEntries)) {
      throw new IOException("Coprocessor bypassing region "
          + this.parent.getRegionNameAsString() + " split.");
    }
    try {
      for (Mutation p : metaEntries) {
        HRegionInfo.parseRegionName(p.getRow());
      }
    } catch (IOException e) {
      LOG.error("Row key of mutation from coprossor is not parsable as region name."
          + "Mutations from coprocessor should only for hbase:meta table.");
      throw e;
    }
  }

  // This is the point of no return.  Adding subsequent edits to .META. as we
  // do below when we do the daughter opens adding each to .META. can fail in
  // various interesting ways the most interesting of which is a timeout
  // BUT the edits all go through (See HBASE-3872).  IF we reach the PONR
  // then subsequent failures need to crash out this regionserver; the
  // server shutdown processing should be able to fix-up the incomplete split.
  // The offlined parent will have the daughters as extra columns.  If
  // we leave the daughter regions in place and do not remove them when we
  // crash out, then they will have their references to the parent in place
  // still and the server shutdown fixup of .META. will point to these
  // regions.
  // We should add PONR JournalEntry before offlineParentInMeta,so even if
  // OfflineParentInMeta timeout,this will cause regionserver exit,and then
  // master ServerShutdownHandler will fix daughter & avoid data loss. (See
  // HBase-4562).
  this.journal.add(JournalEntry.PONR);

  // Edit parent in meta.  Offlines parent region and adds splita and splitb
  // as an atomic update. See HBASE-7721. This update to META makes the region
  // will determine whether the region is split or not in case of failures.
  // If it is successful, master will roll-forward, if not, master will rollback
  // and assign the parent region.
  if (!testing) {
    if (metaEntries == null || metaEntries.isEmpty()) {
      MetaTableAccessor.splitRegion(server.getConnection(), parent.getRegionInfo(),
              daughterRegions.getFirst().getRegionInfo(),
              daughterRegions.getSecond().getRegionInfo(), server.getServerName(), 1);
    } else {
      offlineParentInMetaAndputMetaEntries(server.getConnection(),
        parent.getRegionInfo(), daughterRegions.getFirst().getRegionInfo(), daughterRegions
            .getSecond().getRegionInfo(), server.getServerName(), metaEntries);
    }
  }
  return daughterRegions;
}
 

/**
 * Returns the daughter regions by reading the corresponding columns of the catalog table Result.
 * @param data a Result object from the catalog table scan
 * @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
 */
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
  RegionInfo splitA = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
  RegionInfo splitB = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
  return new PairOfSameType<>(splitA, splitB);
}
 

/**
 * Blocks until the region split is complete in hbase:meta and region server opens the daughters
 */
public static void blockUntilRegionSplit(Configuration conf, long timeout,
    final byte[] regionName, boolean waitForDaughters) throws IOException, InterruptedException {
  long start = System.currentTimeMillis();
  log("blocking until region is split:" + Bytes.toStringBinary(regionName));
  RegionInfo daughterA = null, daughterB = null;
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
    Result result = null;
    RegionInfo region = null;
    while ((System.currentTimeMillis() - start) < timeout) {
      result = metaTable.get(new Get(regionName));
      if (result == null) {
        break;
      }

      region = CatalogFamilyFormat.getRegionInfo(result);
      if (region.isSplitParent()) {
        log("found parent region: " + region.toString());
        PairOfSameType<RegionInfo> pair = MetaTableAccessor.getDaughterRegions(result);
        daughterA = pair.getFirst();
        daughterB = pair.getSecond();
        break;
      }
      Threads.sleep(100);
    }
    if (daughterA == null || daughterB == null) {
      throw new IOException("Failed to get daughters, daughterA=" + daughterA + ", daughterB=" +
        daughterB + ", timeout=" + timeout + ", result=" + result + ", regionName=" +
        Bytes.toString(regionName) + ", region=" + region);
    }

    // if we are here, this means the region split is complete or timed out
    if (waitForDaughters) {
      long rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsInMeta(conn, rem, daughterA);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsInMeta(conn, rem, daughterB);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsOpened(conf, rem, daughterA);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsOpened(conf, rem, daughterB);

      // Compacting the new region to make sure references can be cleaned up
      compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
        TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterA.getRegionName());
      compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
        TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterB.getRegionName());

      removeCompactedFiles(conn, timeout, daughterA);
      removeCompactedFiles(conn, timeout, daughterB);
    }
  }
}
 

@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {

  MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);

  // masterWrapper can be null because this function is called inside of init.
  // If the master is already stopped or has initiated a shutdown, no point in registering the
  // metrics again.
  if (masterWrapper != null && masterWrapper.isRunning()) {

    // Pair<online region number, offline region number>
    PairOfSameType<Integer> regionNumberPair = masterWrapper.getRegionCounts();

    metricsRecordBuilder
        .addGauge(Interns.info(MERGE_PLAN_COUNT_NAME, MERGE_PLAN_COUNT_DESC),
            masterWrapper.getMergePlanCount())
        .addGauge(Interns.info(SPLIT_PLAN_COUNT_NAME, SPLIT_PLAN_COUNT_DESC),
            masterWrapper.getSplitPlanCount())
        .addGauge(Interns.info(MASTER_ACTIVE_TIME_NAME,
            MASTER_ACTIVE_TIME_DESC), masterWrapper.getActiveTime())
        .addGauge(Interns.info(MASTER_START_TIME_NAME,
            MASTER_START_TIME_DESC), masterWrapper.getStartTime())
        .addGauge(Interns.info(MASTER_FINISHED_INITIALIZATION_TIME_NAME,
                MASTER_FINISHED_INITIALIZATION_TIME_DESC),
                masterWrapper.getMasterInitializationTime())
        .addGauge(Interns.info(AVERAGE_LOAD_NAME, AVERAGE_LOAD_DESC),
            masterWrapper.getAverageLoad())
        .addGauge(Interns.info(ONLINE_REGION_COUNT_NAME, ONLINE_REGION_COUNT_DESC),
            regionNumberPair.getFirst())
        .addGauge(Interns.info(OFFLINE_REGION_COUNT_NAME, OFFLINE_REGION_COUNT_DESC),
            regionNumberPair.getSecond())
        .tag(Interns.info(LIVE_REGION_SERVERS_NAME, LIVE_REGION_SERVERS_DESC),
              masterWrapper.getRegionServers())
        .addGauge(Interns.info(NUM_REGION_SERVERS_NAME,
            NUMBER_OF_REGION_SERVERS_DESC), masterWrapper.getNumRegionServers())
        .tag(Interns.info(DEAD_REGION_SERVERS_NAME, DEAD_REGION_SERVERS_DESC),
              masterWrapper.getDeadRegionServers())
        .addGauge(Interns.info(NUM_DEAD_REGION_SERVERS_NAME,
            NUMBER_OF_DEAD_REGION_SERVERS_DESC),
            masterWrapper.getNumDeadRegionServers())
        .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC),
            masterWrapper.getZookeeperQuorum())
        .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName())
        .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId())
        .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC),
            String.valueOf(masterWrapper.getIsActiveMaster()));
  }

  metricsRegistry.snapshot(metricsRecordBuilder, all);
  if(metricsAdapter != null) {
    metricsAdapter.snapshotAllMetrics(registry, metricsRecordBuilder);
  }
}
 

/**
 * Run the transaction.
 * @param server Hosting server instance.  Can be null when testing (won't try
 * and update in zk if a null server)
 * @param services Used to online/offline regions.
 * @throws IOException If thrown, transaction failed.
 *          Call {@link #rollback(Server, RegionServerServices)}
 * @return Regions created
 * @throws IOException
 * @see #rollback(Server, RegionServerServices)
 */
public PairOfSameType<HRegion> execute(final Server server,
    final RegionServerServices services)
throws IOException {
  PairOfSameType<HRegion> regions = createDaughters(server, services);
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplitAfterPONR();
  }
  return stepsAfterPONR(server, services, regions);
}
 

/**
 * Returns the daughter regions by reading the corresponding columns of the catalog table
 * Result.
 * (Copied from MetaTableAccessor)
 * @param data a Result object from the catalog table scan
 * @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
 */
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
  RegionInfo splitA = getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
  RegionInfo splitB = getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
  return new PairOfSameType<>(splitA, splitB);
}
 

/**
 * Get the online and offline region counts
 *
 * @return pair of count for online regions and offline regions
 */
PairOfSameType<Integer> getRegionCounts();