Java Code Examples for org.apache.hadoop.hbase.client.RegionInfo#getStartKey()

private void verifyBounds(List<byte[]> expectedBounds, TableName tableName)
        throws Exception {
  // Get region boundaries from the cluster and verify their endpoints
  final int numRegions = expectedBounds.size()-1;
  try (Table table = UTIL.getConnection().getTable(tableName);
      RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) {
    final List<HRegionLocation> regionInfoMap = locator.getAllRegionLocations();
    assertEquals(numRegions, regionInfoMap.size());
    for (HRegionLocation entry : regionInfoMap) {
      final RegionInfo regionInfo = entry.getRegion();
      byte[] regionStart = regionInfo.getStartKey();
      byte[] regionEnd = regionInfo.getEndKey();

      // This region's start key should be one of the region boundaries
      int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart);
      assertNotSame(-1, startBoundaryIndex);

      // This region's end key should be the region boundary that comes
      // after the starting boundary.
      byte[] expectedRegionEnd = expectedBounds.get(startBoundaryIndex + 1);
      assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd));
    }
  }
}
 

public static void deleteFirstIndexRegion(SpliceWatcher spliceWatcher, Connection connection, String schemaName, String tableName, String indexName) throws Exception {
    SConfiguration config = HConfiguration.getConfiguration();
    HBaseTestingUtility testingUtility = new HBaseTestingUtility((Configuration) config.getConfigSource().unwrapDelegate());
    Admin admin = testingUtility.getAdmin();

    // Delete 2nd region of index
    long   conglomerateId = TableSplit.getConglomerateId(connection, schemaName, tableName, indexName);
    TableName iName = TableName.valueOf(config.getNamespace(),Long.toString(conglomerateId));
    List<RegionInfo> partitions = admin.getRegions(iName);
    for (RegionInfo partition : partitions) {
        byte[] startKey = partition.getStartKey();
        if (startKey.length == 0) {
            String encodedRegionName = partition.getEncodedName();
            spliceWatcher.execute(String.format("call syscs_util.delete_region('%s', '%s', '%s', '%s', false)",
                    schemaName, tableName, indexName, encodedRegionName));
            break;
        }
    }
}
 

public static void blockUntilRegionIsOpened(Configuration conf, long timeout, RegionInfo hri)
    throws IOException, InterruptedException {
  log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
  long start = System.currentTimeMillis();
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table table = conn.getTable(hri.getTable())) {
    byte[] row = hri.getStartKey();
    // Check for null/empty row. If we find one, use a key that is likely to be in first region.
    if (row == null || row.length <= 0) {
      row = new byte[] { '0' };
    }
    Get get = new Get(row);
    while (System.currentTimeMillis() - start < timeout) {
      try {
        table.get(get);
        break;
      } catch (IOException ex) {
        // wait some more
      }
      Threads.sleep(100);
    }
  }
}
 

void verifyTableRegions(Set<RegionInfo> regions) {
  log("Verifying " + regions.size() + " regions: " + regions);

  byte[][] startKeys = new byte[regions.size()][];
  byte[][] endKeys = new byte[regions.size()][];

  int i = 0;
  for (RegionInfo region : regions) {
    startKeys[i] = region.getStartKey();
    endKeys[i] = region.getEndKey();
    i++;
  }

  Pair<byte[][], byte[][]> keys = new Pair<>(startKeys, endKeys);
  verifyStartEndKeys(keys);
}
 

private Scan validateKey(final RegionInfo region, final ExportProtos.ExportRequest request)
    throws IOException {
  Scan scan = ProtobufUtil.toScan(request.getScan());
  byte[] regionStartKey = region.getStartKey();
  byte[] originStartKey = scan.getStartRow();
  if (originStartKey == null
          || Bytes.compareTo(originStartKey, regionStartKey) < 0) {
    scan.withStartRow(regionStartKey);
  }
  byte[] regionEndKey = region.getEndKey();
  byte[] originEndKey = scan.getStopRow();
  if (originEndKey == null
          || Bytes.compareTo(originEndKey, regionEndKey) > 0) {
    scan.withStartRow(regionEndKey);
  }
  return scan;
}
 

private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
    int numRegions, int numReplica, Connection connection) throws IOException {
  SnapshotOfRegionAssignmentFromMeta snapshot =
    new SnapshotOfRegionAssignmentFromMeta(connection);
  snapshot.initialize();
  Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
  assert (regionToServerMap.size() == numRegions * numReplica);
  Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
  for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
    if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
      continue;
    }
    List<RegionInfo> regions = entry.getValue();
    Set<byte[]> setOfStartKeys = new HashSet<>();
    for (RegionInfo region : regions) {
      byte[] startKey = region.getStartKey();
      if (region.getTable().equals(table)) {
        setOfStartKeys.add(startKey); // ignore other tables
        LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
      }
    }
    // the number of startkeys will be equal to the number of regions hosted in each server
    // (each server will be hosting one replica of a region)
    assertEquals(numRegions, setOfStartKeys.size());
  }
}
 

private int loadARowPerRegion(final Table t, List<RegionInfo> ris)
    throws IOException {
  List<Put> puts = new ArrayList<>();
  for (RegionInfo ri: ris) {
    Put put = new Put(ri.getStartKey() == null || ri.getStartKey().length == 0?
        new byte [] {'a'}: ri.getStartKey());
    put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY,
        HConstants.CATALOG_FAMILY);
    puts.add(put);
  }
  t.put(puts);
  return puts.size();
}
 

private void updateRegionSplitTransition(final ServerName serverName, final TransitionCode state,
    final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB)
    throws IOException {
  checkMetaLoaded(parent);

  if (state != TransitionCode.READY_TO_SPLIT) {
    throw new UnexpectedStateException("unsupported split regionState=" + state +
      " for parent region " + parent +
      " maybe an old RS (< 2.0) had the operation in progress");
  }

  // sanity check on the request
  if (!Bytes.equals(hriA.getEndKey(), hriB.getStartKey())) {
    throw new UnsupportedOperationException(
      "unsupported split request with bad keys: parent=" + parent +
      " hriA=" + hriA + " hriB=" + hriB);
  }

  if (!master.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
    LOG.warn("Split switch is off! skip split of " + parent);
    throw new DoNotRetryIOException("Split region " + parent.getRegionNameAsString() +
        " failed due to split switch off");
  }

  // Submit the Split procedure
  final byte[] splitKey = hriB.getStartKey();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Split request from " + serverName +
        ", parent=" + parent + " splitKey=" + Bytes.toStringBinary(splitKey));
  }
  master.getMasterProcedureExecutor().submitProcedure(createSplitProcedure(parent, splitKey));

  // If the RS is < 2.0 throw an exception to abort the operation, we are handling the split
  if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) {
    throw new UnsupportedOperationException(String.format(
      "Split handled by the master: parent=%s hriA=%s hriB=%s", parent.getShortNameToLog(), hriA, hriB));
  }
}
 

/**
 * Create merged region info by looking at passed in <code>regionsToMerge</code>
 * to figure what extremes for start and end keys to use; merged region needs
 * to have an extent sufficient to cover all regions-to-merge.
 */
private static RegionInfo createMergedRegionInfo(final RegionInfo[] regionsToMerge) {
  byte [] lowestStartKey = null;
  byte [] highestEndKey = null;
  // Region Id is a timestamp. Merged region's id can't be less than that of
  // merging regions else will insert at wrong location in hbase:meta (See HBASE-710).
  long highestRegionId = -1;
  for (RegionInfo ri: regionsToMerge) {
    if (lowestStartKey == null) {
      lowestStartKey = ri.getStartKey();
    } else if (Bytes.compareTo(ri.getStartKey(), lowestStartKey) < 0) {
      lowestStartKey = ri.getStartKey();
    }
    if (highestEndKey == null) {
      highestEndKey = ri.getEndKey();
    } else if (ri.isLast() || Bytes.compareTo(ri.getEndKey(), highestEndKey) > 0) {
      highestEndKey = ri.getEndKey();
    }
    highestRegionId = ri.getRegionId() > highestRegionId? ri.getRegionId(): highestRegionId;
  }
  // Merged region is sorted between two merging regions in META
  return RegionInfoBuilder.newBuilder(regionsToMerge[0].getTable()).
      setStartKey(lowestStartKey).
      setEndKey(highestEndKey).
      setSplit(false).
      setRegionId(highestRegionId + 1/*Add one so new merged region is highest*/).
      build();
}
 

@Override
public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException {
  byte[] startKey = RegionInfo.getStartKey(regionName);
  HRegionLocation regionLoc = null;
  try (RegionLocator locator = connection.getRegionLocator(tn)) {
    regionLoc = locator.getRegionLocation(startKey, true);
  }
  if (regionLoc == null) {
    LOG.warn("Cannot find region server holding region {}", Bytes.toStringBinary(regionName));
    return null;
  }
  return regionLoc.getServerName();
}
 

void checkTableInfo(TableInfoModel model) {
  assertEquals(model.getName(), TABLE.getNameAsString());
  Iterator<TableRegionModel> regions = model.getRegions().iterator();
  assertTrue(regions.hasNext());
  while (regions.hasNext()) {
    TableRegionModel region = regions.next();
    boolean found = false;
    LOG.debug("looking for region " + region.getName());
    for (HRegionLocation e: regionMap) {
      RegionInfo hri = e.getRegion();
      // getRegionNameAsString uses Bytes.toStringBinary which escapes some non-printable
      // characters
      String hriRegionName = Bytes.toString(hri.getRegionName());
      String regionName = region.getName();
      LOG.debug("comparing to region " + hriRegionName);
      if (hriRegionName.equals(regionName)) {
        found = true;
        byte[] startKey = hri.getStartKey();
        byte[] endKey = hri.getEndKey();
        ServerName serverName = e.getServerName();
        InetSocketAddress sa =
            new InetSocketAddress(serverName.getHostname(), serverName.getPort());
        String location = sa.getHostName() + ":" +
          Integer.valueOf(sa.getPort());
        assertEquals(hri.getRegionId(), region.getId());
        assertTrue(Bytes.equals(startKey, region.getStartKey()));
        assertTrue(Bytes.equals(endKey, region.getEndKey()));
        assertEquals(location, region.getLocation());
        break;
      }
    }
    assertTrue("Couldn't find region " + region.getName(), found);
  }
}
 

public static byte[] getRowForRegion(RegionInfo hri) {
  byte[] startKey = hri.getStartKey();
  if (startKey.length == 0) {
    // empty row key is not allowed in mutations because it is both the start key and the end key
    // we return the smallest byte[] that is bigger (in lex comparison) than byte[0].
    return new byte[] {0};
  }
  return startKey;
}
 

private static byte [] getStartKey(final RegionInfo hri) {
  return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
      Bytes.toBytes("aaa"): hri.getStartKey();
}
 

@Test
public void testInterClusterReplication() throws Exception {
  final String id = "testInterClusterReplication";

  List<HRegion> regions = UTIL1.getHBaseCluster().getRegions(tableName);
  int totEdits = 0;

  // Make sure edits are spread across regions because we do region based batching
  // before shipping edits.
  for(HRegion region: regions) {
    RegionInfo hri = region.getRegionInfo();
    byte[] row = hri.getStartKey();
    for (int i = 0; i < 100; i++) {
      if (row.length > 0) {
        Put put = new Put(row);
        put.addColumn(famName, row, row);
        region.put(put);
        totEdits++;
      }
    }
  }

  hbaseAdmin.addReplicationPeer(id,
      new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(CONF2))
          .setReplicationEndpointImpl(InterClusterReplicationEndpointForTest.class.getName()));

  final int numEdits = totEdits;
  Waiter.waitFor(CONF1, 30000, new Waiter.ExplainingPredicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return InterClusterReplicationEndpointForTest.replicateCount.get() == numEdits;
    }

    @Override
    public String explainFailure() throws Exception {
      String failure = "Failed to replicate all edits, expected = " + numEdits
          + " replicated = " + InterClusterReplicationEndpointForTest.replicateCount.get();
      return failure;
    }
  });

  hbaseAdmin.removeReplicationPeer("testInterClusterReplication");
  UTIL1.deleteTableData(tableName);
}
 

public void checkRegionBoundaries() {
  try {
    ByteArrayComparator comparator = new ByteArrayComparator();
    List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
    final RegionBoundariesInformation currentRegionBoundariesInformation =
        new RegionBoundariesInformation();
    Path hbaseRoot = CommonFSUtils.getRootDir(getConf());
    for (RegionInfo regionInfo : regions) {
      Path tableDir = CommonFSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
      currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
      // For each region, get the start and stop key from the META and compare them to the
      // same information from the Stores.
      Path path = new Path(tableDir, regionInfo.getEncodedName());
      FileSystem fs = path.getFileSystem(getConf());
      FileStatus[] files = fs.listStatus(path);
      // For all the column families in this region...
      byte[] storeFirstKey = null;
      byte[] storeLastKey = null;
      for (FileStatus file : files) {
        String fileName = file.getPath().toString();
        fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
        if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
          FileStatus[] storeFiles = fs.listStatus(file.getPath());
          // For all the stores in this column family.
          for (FileStatus storeFile : storeFiles) {
            HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(),
              CacheConfig.DISABLED, true, getConf());
            if ((reader.getFirstKey() != null)
                && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
                    ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) {
              storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey().get()).getKey();
            }
            if ((reader.getLastKey() != null)
                && ((storeLastKey == null) || (comparator.compare(storeLastKey,
                    ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey())) < 0)) {
              storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey();
            }
            reader.close();
          }
        }
      }
      currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
      currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
      currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
      currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
      if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
        currentRegionBoundariesInformation.metaFirstKey = null;
      if (currentRegionBoundariesInformation.metaLastKey.length == 0)
        currentRegionBoundariesInformation.metaLastKey = null;

      // For a region to be correct, we need the META start key to be smaller or equal to the
      // smallest start key from all the stores, and the start key from the next META entry to
      // be bigger than the last key from all the current stores. First region start key is null;
      // Last region end key is null; some regions can be empty and not have any store.

      boolean valid = true;
      // Checking start key.
      if ((currentRegionBoundariesInformation.storesFirstKey != null)
          && (currentRegionBoundariesInformation.metaFirstKey != null)) {
        valid = valid
            && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
              currentRegionBoundariesInformation.metaFirstKey) >= 0;
      }
      // Checking stop key.
      if ((currentRegionBoundariesInformation.storesLastKey != null)
          && (currentRegionBoundariesInformation.metaLastKey != null)) {
        valid = valid
            && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
              currentRegionBoundariesInformation.metaLastKey) < 0;
      }
      if (!valid) {
        errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
          tablesInfo.get(regionInfo.getTable()));
        LOG.warn("Region's boundaries not aligned between stores and META for:");
        LOG.warn(Objects.toString(currentRegionBoundariesInformation));
      }
    }
  } catch (IOException e) {
    LOG.error(e.toString(), e);
  }
}
 

@Test
public void test() throws Exception {
  TableName tableName = TableName.valueOf("SplitMerge");
  byte[] family = Bytes.toBytes("CF");
  TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
  UTIL.getAdmin().createTable(td, new byte[][] { Bytes.toBytes(1) });
  UTIL.waitTableAvailable(tableName);
  UTIL.getAdmin().split(tableName, Bytes.toBytes(2));
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 3;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Split has not finished yet";
    }
  });
  UTIL.waitUntilNoRegionsInTransition();
  RegionInfo regionA = null;
  RegionInfo regionB = null;
  for (RegionInfo region : UTIL.getAdmin().getRegions(tableName)) {
    if (region.getStartKey().length == 0) {
      regionA = region;
    } else if (Bytes.equals(region.getStartKey(), Bytes.toBytes(1))) {
      regionB = region;
    }
  }
  assertNotNull(regionA);
  assertNotNull(regionB);
  UTIL.getAdmin().mergeRegionsAsync(regionA.getRegionName(), regionB.getRegionName(), false)
    .get(30, TimeUnit.SECONDS);
  assertEquals(2, UTIL.getAdmin().getRegions(tableName).size());

  ServerName expected = UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
  assertEquals(expected, UTIL.getConnection().getRegionLocator(tableName)
    .getRegionLocation(Bytes.toBytes(1), true).getServerName());
  try (AsyncConnection asyncConn =
    ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get()) {
    assertEquals(expected, asyncConn.getRegionLocator(tableName)
      .getRegionLocation(Bytes.toBytes(1), true).get().getServerName());
  }
}
 

public SpliceMessage.PrepareBackupResponse.Builder prepare(SpliceMessage.PrepareBackupRequest request) throws Exception{

        SpliceMessage.PrepareBackupResponse.Builder responseBuilder = SpliceMessage.PrepareBackupResponse.newBuilder();
        responseBuilder.setReadyForBackup(false);

        if (!BackupUtils.regionKeysMatch(request, region)) {
            // if the start/end key of the request does not match this region, return false to the client, because
            // region has been split. The client should retry.
            SpliceLogUtils.info(LOG, "preparing backup for table %s region %s", tableName, regionName);
            SpliceLogUtils.info(LOG, "Region keys do not match with keys in the request");
            return responseBuilder;
        }

        boolean canceled = false;

        long backupId = request.getBackupId();
        String backupJobPath = BackupUtils.getBackupPath() + "/" + backupId;
        String regionBackupPath = backupJobPath + "/" + tableName + "/" + regionName;
        if (isSplitting.get() || isCompacting.get()) {
            SpliceLogUtils.info(LOG, "table %s region %s is not ready for backup: isSplitting=%s, isCompacting=%s",
                    tableName , regionName, isSplitting.get(), isCompacting.get());

            // return false to client if the region is being split
            responseBuilder.setReadyForBackup(false);
        } else {
            if (LOG.isDebugEnabled()) {
                SpliceLogUtils.debug(LOG, "%s:%s waits for flush and compaction to complete", tableName, regionName);
            }

            // A region might have been in backup. This is unlikely to happen unless the previous response was lost
            // and the client is retrying
            if (!BackupUtils.regionIsBeingBackup(tableName, regionName, backupJobPath, regionBackupPath)) {
                // Flush memstore and Wait for flush and compaction to be done
                region.flushcache(false,false, null);
                region.waitForFlushesAndCompactions();

                canceled = BackupUtils.backupCanceled();
                if (!canceled) {
                    // Create a ZNode to indicate that the region is being copied
                    RegionInfo regionInfo = region.getRegionInfo();
                    BackupRegionStatus backupRegionStatus = new BackupRegionStatus(regionInfo.getStartKey(), regionInfo.getEndKey(),
                            HConfiguration.BACKUP_IN_PROGRESS);
                    boolean created = ZkUtils.recursiveSafeCreate(regionBackupPath, backupRegionStatus.toBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                    if (LOG.isDebugEnabled()) {
                        if (ZkUtils.getRecoverableZooKeeper().exists(regionBackupPath, false) != null) {
                            SpliceLogUtils.debug(LOG,"created znode %s to mark backup in progress, created = %s", regionBackupPath, created);
                        }else {
                            SpliceLogUtils.warn(LOG, "failed to create znode %s, created = %s", regionBackupPath, created);
                        }
                    }

                    if (isCompacting.get() || isSplitting.get()) {

                        SpliceLogUtils.info(LOG, "table %s region %s is not ready for backup: isSplitting=%s, isCompacting=%s",
                                tableName, regionName, isSplitting.get(), isCompacting.get());
                        SpliceLogUtils.info(LOG, "delete znode %d", regionBackupPath);

                        ZkUtils.recursiveDelete(regionBackupPath);
                    }
                    else {
                        responseBuilder.setReadyForBackup(true);
                        if (LOG.isDebugEnabled()) {
                            SpliceLogUtils.debug(LOG, "%s:%s is ready for backup", tableName, regionName);
                        }
                    }
                }
            }
            else
                responseBuilder.setReadyForBackup(true);
        }
        return responseBuilder;
    }
 

/**
 * Get the start key for display. Optionally hide the real start key.
 * @param ri
 * @param conf
 * @return the startkey
 */
public static byte[] getStartKeyForDisplay(RegionInfo ri, Configuration conf) {
  boolean displayKey = conf.getBoolean(DISPLAY_KEYS_KEY, true);
  if (displayKey) return ri.getStartKey();
  return HIDDEN_START_KEY;
}