Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegion#openHRegion()

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  TableDescriptorBuilder tableBuilder =
      TableDescriptorBuilder.newBuilder(region.getTableDescriptor());
  ColumnFamilyDescriptorBuilder cfd =
      ColumnFamilyDescriptorBuilder.newBuilder(tableBuilder.build().getColumnFamily(family));
  if (ttl > 0) {
    cfd.setValue(Bytes.toBytes(TxConstants.PROPERTY_TTL), Bytes.toBytes(String.valueOf(ttl)));
  }
  cfd.setMaxVersions(10);
  tableBuilder.removeColumnFamily(family);
  tableBuilder.addColumnFamily(cfd.build());
  return HRegion
      .openHRegion(region.getRegionInfo(), tableBuilder.build(), region.getWAL(), conf,
        new LocalRegionServerServices(conf, ServerName
            .valueOf(InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())),
        null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
                             new MockRegionServerServices(conf, null), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 

private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getLog(), conf,
                                        new MockRegionServerServices(conf, null), null);
}
 

/**
 * @return Instance of HRegion if successful open else null.
 */
private HRegion openRegion() {
  HRegion region = null;
  try {
    // Instantiate the region.  This also periodically tickles OPENING
    // state so master doesn't timeout this region in transition.
    region = HRegion.openHRegion(this.regionInfo, this.htd,
      this.rsServices.getWAL(this.regionInfo),
      this.server.getConfiguration(),
      this.rsServices,
      new CancelableProgressable() {
        @Override
        public boolean progress() {
          if (!isRegionStillOpening()) {
            LOG.warn("Open region aborted since it isn't opening any more");
            return false;
          }
          return true;
        }
      });
  } catch (Throwable t) {
    // We failed open. Our caller will see the 'null' return value
    // and transition the node back to FAILED_OPEN. If that fails,
    // we rely on the Timeout Monitor in the master to reassign.
    LOG.error(
        "Failed open of region=" + this.regionInfo.getRegionNameAsString(), t);
  }
  return region;
}
 

@Test
public void testDifferentRootDirAndWALRootDir() throws Exception {
  // Change wal root dir and reset the configuration
  Path walRootDir = UTIL.createWALRootDir();
  this.conf = HBaseConfiguration.create(UTIL.getConfiguration());

  FileSystem walFs = CommonFSUtils.getWALFileSystem(this.conf);
  this.oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  String serverName =
      ServerName.valueOf(TEST_NAME.getMethodName() + "-manual", 16010, System.currentTimeMillis())
          .toString();
  this.logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
  this.logDir = new Path(walRootDir, logName);
  this.wals = new WALFactory(conf, TEST_NAME.getMethodName());

  Pair<TableDescriptor, RegionInfo> pair = setupTableAndRegion();
  TableDescriptor td = pair.getFirst();
  RegionInfo ri = pair.getSecond();

  WAL wal = createWAL(walFs, walRootDir, logName);
  HRegion region = HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal);
  writeData(td, region);

  // Now close the region without flush
  region.close(true);
  wal.shutdown();
  // split the log
  WALSplitter.split(walRootDir, logDir, oldLogDir, FileSystem.get(this.conf), this.conf, wals);

  WAL wal2 = createWAL(walFs, walRootDir, logName);
  HRegion region2 = HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal2);
  Result result2 = region2.get(new Get(ROW));
  assertEquals(td.getColumnFamilies().length, result2.size());
  for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
    assertTrue(Bytes.equals(VALUE1, result2.getValue(cfd.getName(), QUALIFIER)));
  }
}
 

@Before
public void before() throws IOException {
  String methodName = this.name.getMethodName();
  TableName tn = TableName.valueOf(methodName);
  ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(methodName)).build();
  TableDescriptor td = TableDescriptorBuilder.newBuilder(tn).setColumnFamily(cfd).build();
  RegionInfo ri = RegionInfoBuilder.newBuilder(tn).build();
  this.rss = new MockRegionServerServices(HTU.getConfiguration());
  this.region = HRegion.openHRegion(ri, td, null, HTU.getConfiguration(), this.rss, null);
}
 

HRegion reopenRegion(final HRegion closedRegion, Class<?> ... implClasses)
    throws IOException {
  //RegionInfo info = new RegionInfo(tableName, null, null, false);
  HRegion r = HRegion.openHRegion(closedRegion, null);

  // this following piece is a hack. currently a coprocessorHost
  // is secretly loaded at OpenRegionHandler. we don't really
  // start a region server here, so just manually create cphost
  // and set it to region.
  Configuration conf = TEST_UTIL.getConfiguration();
  RegionCoprocessorHost host = new RegionCoprocessorHost(r,
      Mockito.mock(RegionServerServices.class), conf);
  r.setCoprocessorHost(host);

  for (Class<?> implClass : implClasses) {
    host.load((Class<? extends RegionCoprocessor>) implClass, Coprocessor.PRIORITY_USER, conf);
  }
  // we need to manually call pre- and postOpen here since the
  // above load() is not the real case for CP loading. A CP is
  // expected to be loaded by default from 1) configuration; or 2)
  // HTableDescriptor. If it's loaded after HRegion initialized,
  // the pre- and postOpen() won't be triggered automatically.
  // Here we have to call pre and postOpen explicitly.
  host.preOpen();
  host.postOpen();
  return r;
}
 

/**
 * testcase for https://issues.apache.org/jira/browse/HBASE-14949.
 */
private void testNameConflictWhenSplit(boolean largeFirst) throws IOException,
    StreamLacksCapabilityException {
  final TableName tableName = TableName.valueOf("testReplayEditsWrittenIntoWAL");
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  final RegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
  final Path basedir = CommonFSUtils.getTableDir(hbaseRootDir, tableName);
  deleteDir(basedir);

  final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : htd.getFamiliesKeys()) {
    scopes.put(fam, 0);
  }
  HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, hbaseRootDir, this.conf, htd);
  HBaseTestingUtility.closeRegionAndWAL(region);
  final byte[] family = htd.getColumnFamilies()[0].getName();
  final byte[] rowName = tableName.getName();
  FSWALEntry entry1 = createFSWALEntry(htd, hri, 1L, rowName, family, ee, mvcc, 1, scopes);
  FSWALEntry entry2 = createFSWALEntry(htd, hri, 2L, rowName, family, ee, mvcc, 2, scopes);

  Path largeFile = new Path(logDir, "wal-1");
  Path smallFile = new Path(logDir, "wal-2");
  writerWALFile(largeFile, Arrays.asList(entry1, entry2));
  writerWALFile(smallFile, Arrays.asList(entry2));
  FileStatus first, second;
  if (largeFirst) {
    first = fs.getFileStatus(largeFile);
    second = fs.getFileStatus(smallFile);
  } else {
    first = fs.getFileStatus(smallFile);
    second = fs.getFileStatus(largeFile);
  }
  WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, wals, null);
  WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, wals, null);
  WAL wal = createWAL(this.conf, hbaseRootDir, logName);
  region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal);
  assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint());
  assertEquals(2, region.get(new Get(rowName)).size());
}
 

public SnapshotScanner(Configuration conf, FileSystem fs, Path rootDir,
    TableDescriptor htd, RegionInfo hri,  Scan scan) throws Throwable{

  LOGGER.info("Creating SnapshotScanner for region: " + hri);

  scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
  values = new ArrayList<>();
  this.region = HRegion.openHRegion(conf, fs, rootDir, hri, htd, null, null, null);
  this.scan = scan;

  RegionCoprocessorEnvironment snapshotEnv = getSnapshotContextEnvironment(conf);

  // Collect statistics during scan if ANALYZE_TABLE attribute is set
  if (ScanUtil.isAnalyzeTable(scan)) {
    this.scanner = region.getScanner(scan);
    PhoenixConnection connection = (PhoenixConnection) ConnectionUtil.getInputConnection(conf, new Properties());
    String tableName = region.getTableDescriptor().getTableName().getNameAsString();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, conf);
    Table table = connection.getQueryServices().getTable(physicalTableName.getName());
    StatisticsWriter statsWriter = StatisticsWriter.newWriter(connection, tableName, HConstants.LATEST_TIMESTAMP);
    statisticsCollector = new DefaultStatisticsCollector(conf, region,
            tableName, null, null, null, statsWriter, table);
  } else if (scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null) {
    RegionScannerFactory regionScannerFactory = new NonAggregateRegionScannerFactory(snapshotEnv);
    this.scanner = regionScannerFactory.getRegionScanner(scan, region.getScanner(scan));
    statisticsCollector = new NoOpStatisticsCollector();
  } else {
    /* future work : Snapshot M/R jobs for aggregate queries*/
    throw new UnsupportedOperationException("Snapshot M/R jobs not available for aggregate queries");
  }

  statisticsCollector.init();
  region.startRegionOperation();
}
 

@Override
public void process() throws IOException {
  HRegionServer rs = getServer();
  String encodedName = regionInfo.getEncodedName();
  byte[] encodedNameBytes = regionInfo.getEncodedNameAsBytes();
  String regionName = regionInfo.getRegionNameAsString();
  Region onlineRegion = rs.getRegion(encodedName);
  if (onlineRegion != null) {
    LOG.warn("Received OPEN for the region:{}, which is already online", regionName);
    // Just follow the old behavior, do we need to call reportRegionStateTransition? Maybe not?
    // For normal case, it could happen that the rpc call to schedule this handler is succeeded,
    // but before returning to master the connection is broken. And when master tries again, we
    // have already finished the opening. For this case we do not need to call
    // reportRegionStateTransition any more.
    return;
  }
  Boolean previous = rs.getRegionsInTransitionInRS().putIfAbsent(encodedNameBytes, Boolean.TRUE);
  if (previous != null) {
    if (previous) {
      // The region is opening and this maybe a retry on the rpc call, it is safe to ignore it.
      LOG.info("Receiving OPEN for the region:{}, which we are already trying to OPEN" +
        " - ignoring this new request for this region.", regionName);
    } else {
      // The region is closing. This is possible as we will update the region state to CLOSED when
      // calling reportRegionStateTransition, so the HMaster will think the region is offline,
      // before we actually close the region, as reportRegionStateTransition is part of the
      // closing process.
      long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
      LOG.info(
        "Receiving OPEN for the region:{}, which we are trying to close, try again after {}ms",
        regionName, backoff);
      rs.getExecutorService().delayedSubmit(this, backoff, TimeUnit.MILLISECONDS);
    }
    return;
  }
  LOG.info("Open {}", regionName);
  HRegion region;
  try {
    TableDescriptor htd =
      tableDesc != null ? tableDesc : rs.getTableDescriptors().get(regionInfo.getTable());
    if (htd == null) {
      throw new IOException("Missing table descriptor for " + regionName);
    }
    // pass null for the last parameter, which used to be a CancelableProgressable, as now the
    // opening can not be interrupted by a close request any more.
    region = HRegion.openHRegion(regionInfo, htd, rs.getWAL(regionInfo), rs.getConfiguration(),
      rs, null);
  } catch (IOException e) {
    cleanUpAndReportFailure(e);
    return;
  }
  // From here on out, this is PONR. We can not revert back. The only way to address an
  // exception from here on out is to abort the region server.
  rs.postOpenDeployTasks(new PostOpenDeployContext(region, openProcId, masterSystemTime));
  rs.addRegion(region);
  LOG.info("Opened {}", regionName);
  // Cache the open region procedure id after report region transition succeed.
  rs.finishRegionProcedure(openProcId);
  Boolean current = rs.getRegionsInTransitionInRS().remove(regionInfo.getEncodedNameAsBytes());
  if (current == null) {
    // Should NEVER happen, but let's be paranoid.
    LOG.error("Bad state: we've just opened a region that was NOT in transition. Region={}",
      regionName);
  } else if (!current) {
    // Should NEVER happen, but let's be paranoid.
    LOG.error("Bad state: we've just opened a region that was closing. Region={}", regionName);
  }
}
 

/**
 * Test writing edits into an HRegion, closing it, splitting logs, opening
 * Region again.  Verify seqids.
 */
@Test
public void testWrittenViaHRegion()
    throws IOException, SecurityException, IllegalArgumentException, InterruptedException {
  Pair<TableDescriptor, RegionInfo> pair = setupTableAndRegion();
  TableDescriptor td = pair.getFirst();
  RegionInfo ri = pair.getSecond();

  // Write countPerFamily edits into the three families.  Do a flush on one
  // of the families during the load of edits so its seqid is not same as
  // others to test we do right thing when different seqids.
  WAL wal = createWAL(this.conf, rootDir, logName);
  HRegion region = HRegion.openHRegion(this.conf, this.fs, rootDir, ri, td, wal);
  long seqid = region.getOpenSeqNum();
  boolean first = true;
  for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
    addRegionEdits(ROW, cfd.getName(), countPerFamily, this.ee, region, "x");
    if (first) {
      // If first, so we have at least one family w/ different seqid to rest.
      region.flush(true);
      first = false;
    }
  }
  // Now assert edits made it in.
  final Get g = new Get(ROW);
  Result result = region.get(g);
  assertEquals(countPerFamily * td.getColumnFamilies().length, result.size());
  // Now close the region (without flush), split the log, reopen the region and assert that
  // replay of log has the correct effect, that our seqids are calculated correctly so
  // all edits in logs are seen as 'stale'/old.
  region.close(true);
  wal.shutdown();
  try {
    WALSplitter.split(rootDir, logDir, oldLogDir, FileSystem.get(this.conf), this.conf, wals);
  } catch (Exception e) {
    LOG.debug("Got exception", e);
  }

  WAL wal2 = createWAL(this.conf, rootDir, logName);
  HRegion region2 = HRegion.openHRegion(conf, this.fs, rootDir, ri, td, wal2);
  long seqid2 = region2.getOpenSeqNum();
  assertTrue(seqid + result.size() < seqid2);
  final Result result1b = region2.get(g);
  assertEquals(result.size(), result1b.size());

  // Next test.  Add more edits, then 'crash' this region by stealing its wal
  // out from under it and assert that replay of the log adds the edits back
  // correctly when region is opened again.
  for (ColumnFamilyDescriptor hcd : td.getColumnFamilies()) {
    addRegionEdits(ROW, hcd.getName(), countPerFamily, this.ee, region2, "y");
  }
  // Get count of edits.
  final Result result2 = region2.get(g);
  assertEquals(2 * result.size(), result2.size());
  wal2.sync();
  final Configuration newConf = HBaseConfiguration.create(this.conf);
  User user = HBaseTestingUtility.getDifferentUser(newConf, td.getTableName().getNameAsString());
  user.runAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws Exception {
      WALSplitter.split(rootDir, logDir, oldLogDir, FileSystem.get(conf), conf, wals);
      FileSystem newFS = FileSystem.get(newConf);
      // Make a new wal for new region open.
      WAL wal3 = createWAL(newConf, rootDir, logName);
      Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
      HRegion region3 = new HRegion(tableDir, wal3, newFS, newConf, ri, td, null);
      long seqid3 = region3.initialize();
      Result result3 = region3.get(g);
      // Assert that count of cells is same as before crash.
      assertEquals(result2.size(), result3.size());

      // I can't close wal1.  Its been appropriated when we split.
      region3.close();
      wal3.close();
      return null;
    }
  });
}