org.apache.hadoop.hbase.regionserver.InternalScanner Java Examples

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
                                             TransactionVisibilityState snapshot, Store store,
                                             List<? extends KeyValueScanner> scanners, ScanType type,
                                             long earliestPutTs) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
                  ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }

  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  Scan scan = new Scan();
  // need to see all versions, since we filter out excludes and applications may rely on multiple versions
  scan.setMaxVersions();
  scan.setFilter(
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
          snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));

  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
                          type, store.getSmallestReadPoint(), earliestPutTs);
}
 

@Test
public void testFirstKeyOnlyFilterAndBatch() throws IOException {
  Scan scan = new Scan();
  scan.setFilter(new FirstKeyOnlyFilter());
  scan.setBatch(1);
  InternalScanner scanner = REGION.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  for (int i = 0; i < NUM_ROWS; i++) {
    results.clear();
    scanner.next(results);
    assertEquals(1, results.size());
    Cell cell = results.get(0);
    assertArrayEquals(ROWS[i],
        Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
  }
  assertFalse(scanner.next(results));
  scanner.close();
}
 

@Test
public void testWhileMatchFilterWithFilterRowKeyWithReverseScan()
    throws Exception {
  Scan s = new Scan();
  String prefix = "testRowOne";
  WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(
      Bytes.toBytes(prefix)));
  s.setFilter(filter);
  s.setReversed(true);

  InternalScanner scanner = this.region.getScanner(s);
  while (true) {
    ArrayList<Cell> values = new ArrayList<>();
    boolean isMoreResults = scanner.next(values);
    if (!isMoreResults
        || !Bytes.toString(CellUtil.cloneRow(values.get(0))).startsWith(prefix)) {
      Assert.assertTrue(
          "The WhileMatchFilter should now filter all remaining",
          filter.filterAllRemaining());
    }
    if (!isMoreResults) {
      break;
    }
  }
  scanner.close();
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

/**
 * Tests the the {@link WhileMatchFilter} works in combination with a
 * {@link Filter} that uses the
 * {@link Filter#filterRowKey(Cell)} method.
 *
 * See HBASE-2258.
 *
 * @throws Exception
 */
@Test
public void testWhileMatchFilterWithFilterRowKey() throws Exception {
  Scan s = new Scan();
  String prefix = "testRowOne";
  WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(Bytes.toBytes(prefix)));
  s.setFilter(filter);

  InternalScanner scanner = this.region.getScanner(s);
  while (true) {
    ArrayList<Cell> values = new ArrayList<>();
    boolean isMoreResults = scanner.next(values);
    if (!isMoreResults || !Bytes.toString(CellUtil.cloneRow(values.get(0))).startsWith(prefix)) {
      assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
    }
    if (!isMoreResults) {
      break;
    }
  }
}
 

@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
                                             TransactionVisibilityState snapshot, Store store,
                                             List<? extends KeyValueScanner> scanners, ScanType type,
                                             long earliestPutTs) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
                  ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }

  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  Scan scan = new Scan();
  // need to see all versions, since we filter out excludes and applications may rely on multiple versions
  scan.setMaxVersions();
  scan.setFilter(
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
          snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));

  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
                          type, store.getSmallestReadPoint(), earliestPutTs);
}
 

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
                                             TransactionVisibilityState snapshot, Store store,
                                             List<? extends KeyValueScanner> scanners, ScanType type,
                                             long earliestPutTs) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString() +
                  ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }

  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  Scan scan = new Scan();
  // need to see all versions, since we filter out excludes and applications may rely on multiple versions
  scan.setMaxVersions();
  scan.setFilter(
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
          snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));

  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
                          type, store.getSmallestReadPoint(), earliestPutTs);
}
 

/**
 * Tests the the {@link WhileMatchFilter} works in combination with a
 * {@link Filter} that uses the
 * {@link Filter#filterRow()} method.
 *
 * See HBASE-2258.
 *
 * @throws Exception
 */
@Test
public void testWhileMatchFilterWithFilterRow() throws Exception {
  final int pageSize = 4;

  Scan s = new Scan();
  WhileMatchFilter filter = new WhileMatchFilter(new PageFilter(pageSize));
  s.setFilter(filter);

  InternalScanner scanner = this.region.getScanner(s);
  int scannerCounter = 0;
  while (true) {
    boolean isMoreResults = scanner.next(new ArrayList<>());
    scannerCounter++;

    if (scannerCounter >= pageSize) {
      assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
    }
    if (!isMoreResults) {
      break;
    }
  }
  assertEquals("The page filter returned more rows than expected", pageSize, scannerCounter);
}
 

public List<Path> compact(final CompactionRequestImpl request, final List<Long> lowerBoundaries,
    final Map<Long, String> lowerBoundariesPolicies,
    ThroughputController throughputController, User user) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Executing compaction with " + lowerBoundaries.size()
        + "windows, lower boundaries: " + lowerBoundaries);
  }

  return compact(request, defaultScannerFactory,
    new CellSinkFactory<DateTieredMultiFileWriter>() {

      @Override
      public DateTieredMultiFileWriter createWriter(InternalScanner scanner, FileDetails fd,
          boolean shouldDropBehind) throws IOException {
        DateTieredMultiFileWriter writer = new DateTieredMultiFileWriter(lowerBoundaries,
            lowerBoundariesPolicies,
            needEmptyFile(request));
        initMultiWriter(writer, scanner, fd, shouldDropBehind);
        return writer;
      }
    }, throughputController, user);
}
 

/**
 * Do a small get/scan against one store. This is required because store
 * has no actual methods of querying itself, and relies on StoreScanner.
 */
public static List<Cell> getFromStoreFile(HStore store,
                                              Get get) throws IOException {
  Scan scan = new Scan(get);
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
      // originally MultiVersionConcurrencyControl.resetThreadReadPoint() was called to set
      // readpoint 0.
      0);

  List<Cell> result = new ArrayList<>();
  scanner.next(result);
  if (!result.isEmpty()) {
    // verify that we are on the row we want:
    Cell kv = result.get(0);
    if (!CellUtil.matchingRows(kv, get.getRow())) {
      result.clear();
    }
  }
  scanner.close();
  return result;
}
 

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
    TransactionVisibilityState snapshot, InternalScanner scanner,
    ScanType type) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString()
          + ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }
  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  return new FilteredInternalScanner(scanner,
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));
}
 

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
                                             TransactionVisibilityState snapshot, Store store,
                                             List<? extends KeyValueScanner> scanners, ScanType type,
                                             long earliestPutTs) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionNameAsString() +
                  ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }

  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  Scan scan = new Scan();
  // need to see all versions, since we filter out excludes and applications may rely on multiple versions
  scan.setMaxVersions();
  scan.setFilter(
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
          snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));

  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
                          type, store.getSmallestReadPoint(), earliestPutTs);
}
 

/**
 * Tests the the {@link WhileMatchFilter} works in combination with a
 * {@link Filter} that uses the {@link Filter#filterCell(Cell)} method.
 *
 * See HBASE-2258.
 *
 * @throws Exception
 */
@Test
public void testWhileMatchFilterWithFilterCell() throws Exception {
  Scan s = new Scan();
  WhileMatchFilter filter = new WhileMatchFilter(
      new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOperator.EQUAL, Bytes.toBytes("foo"))
  );
  s.setFilter(filter);

  InternalScanner scanner = this.region.getScanner(s);
  while (true) {
    ArrayList<Cell> values = new ArrayList<>();
    boolean isMoreResults = scanner.next(values);
    assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
    if (!isMoreResults) {
      break;
    }
  }
}
 

@Override
public InternalScanner preCompact(
    final ObserverContext<RegionCoprocessorEnvironment> e,
    final Store store,
    final InternalScanner scanner,
    final ScanType scanType,
    final CompactionRequest request) throws IOException {
  if (opStore == null) {
    return super.preCompact(e, store, scanner, scanType, request);
  }
  return super.preCompact(
      e,
      store,
      wrapScannerWithOps(
          e.getEnvironment().getRegionInfo().getTable(),
          scanner,
          null,
          ServerOpScope.MAJOR_COMPACTION,
          INTERNAL_SCANNER_FACTORY),
      scanType,
      request);
}
 

protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action,
                                             TransactionVisibilityState snapshot, Store store,
                                             List<? extends KeyValueScanner> scanners, ScanType type,
                                             long earliestPutTs) throws IOException {
  if (snapshot == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString() +
                  ", no current transaction state found, defaulting to normal " + action + " scanner");
    }
    return null;
  }

  // construct a dummy transaction from the latest snapshot
  Transaction dummyTx = TxUtils.createDummyTransaction(snapshot);
  Scan scan = new Scan();
  // need to see all versions, since we filter out excludes and applications may rely on multiple versions
  scan.setMaxVersions();
  scan.setFilter(
      new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(),
          snapshot.getInvalid(),
          getTransactionFilter(dummyTx, type, null)));

  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
                          type, store.getSmallestReadPoint(), earliestPutTs);
}
 

protected void initMultiWriter(AbstractMultiFileWriter writer, InternalScanner scanner,
    final FileDetails fd, final boolean shouldDropBehind) {
  WriterFactory writerFactory = new WriterFactory() {
    @Override
    public StoreFileWriter createWriter() throws IOException {
      return createTmpWriter(fd, shouldDropBehind);
    }

    @Override
    public StoreFileWriter createWriterWithStoragePolicy(String fileStoragePolicy)
        throws IOException {
      return createTmpWriter(fd, shouldDropBehind, fileStoragePolicy);
    }
  };
  // Prepare multi-writer, and perform the compaction using scanner and writer.
  // It is ok here if storeScanner is null.
  StoreScanner storeScanner = (scanner instanceof StoreScanner) ? (StoreScanner) scanner : null;
  writer.init(storeScanner, writerFactory);
}
 

public List<Path> compact(CompactionRequestImpl request, final List<byte[]> targetBoundaries,
    final byte[] majorRangeFromRow, final byte[] majorRangeToRow,
    ThroughputController throughputController, User user) throws IOException {
  if (LOG.isDebugEnabled()) {
    StringBuilder sb = new StringBuilder();
    sb.append("Executing compaction with " + targetBoundaries.size() + " boundaries:");
    for (byte[] tb : targetBoundaries) {
      sb.append(" [").append(Bytes.toString(tb)).append("]");
    }
    LOG.debug(sb.toString());
  }
  return compact(request, new StripeInternalScannerFactory(majorRangeFromRow, majorRangeToRow),
    new CellSinkFactory<StripeMultiFileWriter>() {

      @Override
      public StripeMultiFileWriter createWriter(InternalScanner scanner, FileDetails fd,
          boolean shouldDropBehind) throws IOException {
        StripeMultiFileWriter writer = new StripeMultiFileWriter.BoundaryMultiWriter(
            store.getComparator(), targetBoundaries, majorRangeFromRow, majorRangeToRow);
        initMultiWriter(writer, scanner, fd, shouldDropBehind);
        return writer;
      }
    }, throughputController, user);
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  requirePermission(c, "compact", getTableName(c.getEnvironment()),
      null, null, Action.ADMIN, Action.CREATE);
  return scanner;
}
 

public StatisticsScanner(StatisticsCollector tracker, StatisticsWriter stats, RegionCoprocessorEnvironment env,
        InternalScanner delegate, ImmutableBytesPtr family) {
    this.tracker = tracker;
    this.statsWriter = stats;
    this.delegate = delegate;
    this.region = env.getRegion();
    this.env = env;
    this.family = family;
    this.config = env.getConfiguration();
    StatisticsCollectionRunTracker.getInstance(config).addCompactingRegion(region.getRegionInfo());
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
    Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  preCompactCalled = true;
  return scanner;
}
 

@Override
public boolean preScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
    final InternalScanner s, final List<Result> result,
    final int limit, final boolean hasNext) throws IOException {
  requireScannerOwner(s);
  return hasNext;
}
 

private void verifyScanFull(Scan s, KeyValue [] kvs)
throws IOException {
  InternalScanner scanner = this.region.getScanner(s);
  List<Cell> results = new ArrayList<>();
  int row = 0;
  int idx = 0;
  for (boolean done = true; done; row++) {
    done = scanner.next(results);
    Arrays.sort(results.toArray(new Cell[results.size()]),
        CellComparator.getInstance());
    if(results.isEmpty()) break;
    assertTrue("Scanned too many keys! Only expected " + kvs.length +
        " total but already scanned " + (results.size() + idx) +
        (results.isEmpty() ? "" : "(" + results.get(0).toString() + ")"),
        kvs.length >= idx + results.size());
    for (Cell kv : results) {
      LOG.info("row=" + row + ", result=" + kv.toString() +
          ", match=" + kvs[idx].toString());
      assertTrue("Row mismatch", CellUtil.matchingRows(kv, kvs[idx]));
      assertTrue("Family mismatch", CellUtil.matchingFamily(kv, kvs[idx]));
      assertTrue("Qualifier mismatch", CellUtil.matchingQualifier(kv, kvs[idx]));
      assertTrue("Value mismatch", CellUtil.matchingValue(kv, kvs[idx]));
      idx++;
    }
    results.clear();
  }
  LOG.info("Looked at " + row + " rows with " + idx + " keys");
  assertEquals("Expected " + kvs.length + " total keys but scanned " + idx,
      kvs.length, idx);
}
 

@Override
public void getRowCount(RpcController controller,
                        getRowCountRequest request,
                        RpcCallback<getRowCountResponse> done) {
    // 单个region上的计算结果值
    int result = 0;

    // 定义返回response
    getRowCountResponse.Builder responseBuilder = getRowCountResponse.newBuilder();
    // 进行行数统计
    InternalScanner scanner = null;
    try {
        Scan scan = new Scan();
        scanner = this.envi.getRegion().getScanner(scan);
        List<Cell> results = new ArrayList<Cell>();
        boolean hasMore = false;

        do {
            hasMore = scanner.next(results);
            result++;
        } while (hasMore);
    } catch (IOException ioe) {
        ioe.printStackTrace();
    } finally {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (IOException ignored) {
                // nothing to do
            }
        }
    }

    responseBuilder.setRowCount(result);
    done.run(responseBuilder.build());
    return;

}
 

public int countRows(final Region region, final Scan scan) throws IOException {
  InternalScanner scanner = region.getScanner(scan);
  try {
    return countRows(scanner);
  } finally {
    scanner.close();
  }
}
 

public InternalScanner createCompactionScanner(HRegion region, Store store, InternalScanner s) throws IOException {
    // See if this is for Major compaction
    if (logger.isDebugEnabled()) {
        logger.debug("Compaction scanner created for stats");
    }
    ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
    return getInternalScanner(region, store, s, cfKey);
}
 

public void verifyInvocationResults(Integer[] selectQualifiers,
    Integer[] expectedQualifiers) throws Exception {
  Get get = new Get(ROW_BYTES);
  for (int i = 0; i < selectQualifiers.length; i++) {
    get.addColumn(FAMILY_NAME_BYTES,
        Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
  }

  get.setFilter(new InvocationRecordFilter());

  List<KeyValue> expectedValues = new ArrayList<>();
  for (int i = 0; i < expectedQualifiers.length; i++) {
    expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes
        .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
        expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX
            + expectedQualifiers[i])));
  }

  Scan scan = new Scan(get);
  List<Cell> actualValues = new ArrayList<>();
  List<Cell> temp = new ArrayList<>();
  InternalScanner scanner = this.region.getScanner(scan);
  while (scanner.next(temp)) {
    actualValues.addAll(temp);
    temp.clear();
  }
  actualValues.addAll(temp);
  Assert.assertTrue("Actual values " + actualValues
      + " differ from the expected values:" + expectedValues,
      expectedValues.equals(actualValues));
}
 

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
    final Store store, InternalScanner scanner, final ScanType scanType)
    throws IOException {
    TableName table = c.getEnvironment().getRegion().getRegionInfo().getTable();
    InternalScanner internalScanner = scanner;
    if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
        try {
            boolean useCurrentTime = 
                    c.getEnvironment().getConfiguration().getBoolean(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB, 
                            QueryServicesOptions.DEFAULT_STATS_USE_CURRENT_TIME);
            // Provides a means of clients controlling their timestamps to not use current time
            // when background tasks are updating stats. Instead we track the max timestamp of
            // the cells and use that.
            long clientTimeStamp = useCurrentTime ? TimeKeeper.SYSTEM.getCurrentTime() : StatisticsCollector.NO_TIMESTAMP;
            StatisticsCollector stats = new StatisticsCollector(
                    c.getEnvironment(), table.getNameAsString(),
                    clientTimeStamp, store.getFamily().getName());
            internalScanner = stats.createCompactionScanner(c.getEnvironment().getRegion(), store, scanner);
        } catch (IOException e) {
            // If we can't reach the stats table, don't interrupt the normal
            // compaction operation, just log a warning.
            if(logger.isWarnEnabled()) {
                logger.warn("Unable to collect stats for " + table, e);
            }
        }
    }
    return internalScanner;
}
 

@Override
public StoreFileWriter createWriter(InternalScanner scanner,
    org.apache.hadoop.hbase.regionserver.compactions.Compactor.FileDetails fd,
    boolean shouldDropBehind) throws IOException {
  // make this writer with tags always because of possible new cells with tags.
  return store.createWriterInTmp(fd.maxKeyCount, compactionCompression, true, true, true,
    shouldDropBehind);
}
 

void checkScannerClose(final InternalScanner s) throws IOException {
  if (s == null) {
    return;
  }
  if (region.getCoprocessorHost() == null) {
    s.close();
    return;
  }
  region.getCoprocessorHost().preScannerClose(s);
  try {
    s.close();
  } finally {
    region.getCoprocessorHost().postScannerClose(s);
  }
}