Java Code Examples for org.apache.hadoop.hbase.client.Scan#setAllowPartialResults()

/**
 * @return The approximate heap size of a cell in the test table. All cells should have
 *         approximately the same heap size, so the value is cached to avoid repeating the
 *         calculation
 * @throws Exception on unexpected failure
 */
private long getCellHeapSize() throws Exception {
  if (CELL_HEAP_SIZE == -1) {
    // Do a partial scan that will return a single result with a single cell
    Scan scan = new Scan();
    scan.setMaxResultSize(1);
    scan.setAllowPartialResults(true);
    ResultScanner scanner = TABLE.getScanner(scan);

    Result result = scanner.next();

    assertTrue(result != null);
    assertTrue(result.rawCells() != null);
    assertTrue(result.rawCells().length == 1);

    CELL_HEAP_SIZE = result.rawCells()[0].heapSize();
    scanner.close();
  }

  return CELL_HEAP_SIZE;
}
 

/**
 * @return The approximate heap size of a cell in the test table. All cells should have
 *         approximately the same heap size, so the value is cached to avoid repeating the
 *         calculation
 * @throws Exception
 */
private long getCellHeapSize() throws Exception {
  if (CELL_HEAP_SIZE == -1) {
    // Do a partial scan that will return a single result with a single cell
    Scan scan = new Scan();
    scan.setMaxResultSize(2);
    scan.setAllowPartialResults(true);
    ResultScanner scanner = TABLE.getScanner(scan);

    Result result = scanner.next();

    assertTrue(result != null);
    assertTrue(result.rawCells() != null);
    assertTrue(result.rawCells().length == 1);

    // Estimate the cell heap size. One difference is that on server side, the KV Heap size is
    // estimated differently in case the cell is backed up by MSLAB byte[] (no overhead for
    // backing array). Thus below calculation is a bit brittle.
    CELL_HEAP_SIZE = result.rawCells()[0].heapSize() - (ClassSize.ARRAY + 3);
    if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE);
    scanner.close();
  }

  return CELL_HEAP_SIZE;
}
 

/**
 * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is
 * true, the scanner should not return partial results. The scanner cannot return partial results
 * because the entire row needs to be read for the include/exclude decision to be made
 */
@Test
public void testNoPartialResultsWhenRowFilterPresent() throws Exception {
  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  // If a filter hasFilter() is true then partial results should not be returned else filter
  // application server side would break.
  scan.setFilter(new RandomRowFilter(1.0f));
  ResultScanner scanner = TABLE.getScanner(scan);

  Result r = null;
  while ((r = scanner.next()) != null) {
    assertFalse(r.mayHaveMoreCellsInRow());
  }

  scanner.close();
}
 

/**
 * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
 * @param cachingRowLimit The row limit that will be enforced through caching
 */
public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
    throws Exception {
  Scan scan = new Scan();
  scan.setAllowPartialResults(true);

  // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
  // ensure that the result size we specify is not an exact multiple of the number of cells
  // in a row. This ensures that partial results will be returned when the result size limit
  // is reached before the caching limit.
  int cellOffset = NUM_COLS / 3;
  long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
  scan.setMaxResultSize(maxResultSize);
  scan.setCaching(cachingRowLimit);

  try (ResultScanner scanner = TABLE.getScanner(scan)) {
    Result r = null;
    // Approximate the number of rows we expect will fit into the specified max rsult size. If
    // this approximation is less than caching, then we expect that the max result size limit will
    // be hit before the caching limit and thus partial results may be seen
    boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
    while ((r = scanner.next()) != null) {
      assertTrue(!r.mayHaveMoreCellsInRow() || expectToSeePartialResults);
    }
  }
}
 

/**
 * Ensure that we only see Results marked as partial when the allowPartial flag is set
 * @throws Exception
 */
@Test
public void testAllowPartialResults() throws Exception {
  Scan scan = new Scan();
  scan.setAllowPartialResults(true);
  scan.setMaxResultSize(1);
  ResultScanner scanner = TABLE.getScanner(scan);
  Result result = scanner.next();

  assertTrue(result != null);
  assertTrue(result.mayHaveMoreCellsInRow());
  assertTrue(result.rawCells() != null);
  assertTrue(result.rawCells().length == 1);

  scanner.close();

  scan.setAllowPartialResults(false);
  scanner = TABLE.getScanner(scan);
  result = scanner.next();

  assertTrue(result != null);
  assertTrue(!result.mayHaveMoreCellsInRow());
  assertTrue(result.rawCells() != null);
  assertTrue(result.rawCells().length == NUM_COLS);

  scanner.close();
}
 

public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells)
    throws Exception {

  if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells);

  // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back
  // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less
  // than it in cases where expectedNumberOfCells is not an exact multiple of the number of
  // columns in the table.
  Scan scan = new Scan(baseScan);
  scan.setAllowPartialResults(true);
  scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells));

  ResultScanner scanner = TABLE.getScanner(scan);
  Result result = null;
  byte[] prevRow = null;
  while ((result = scanner.next()) != null) {
    assertTrue(result.rawCells() != null);

    // Cases when cell count won't equal expectedNumberOfCells:
    // 1. Returned result is the final result needed to form the complete result for that row
    // 2. It is the first result we have seen for that row and thus may have been fetched as
    // the last group of cells that fit inside the maxResultSize
    assertTrue(
        "Result's cell count differed from expected number. result: " + result,
        result.rawCells().length == expectedNumberOfCells || !result.mayHaveMoreCellsInRow()
            || !Bytes.equals(prevRow, result.getRow()));
    prevRow = result.getRow();
  }

  scanner.close();
}
 

public void testPartialResultsReassembly(Scan scanBase) throws Exception {
  Scan partialScan = new Scan(scanBase);
  partialScan.setMaxResultSize(1);
  partialScan.setAllowPartialResults(true);
  ResultScanner partialScanner = TABLE.getScanner(partialScan);

  Scan oneShotScan = new Scan(scanBase);
  oneShotScan.setMaxResultSize(Long.MAX_VALUE);
  ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);

  ArrayList<Result> partials = new ArrayList<>();
  for (int i = 0; i < NUM_ROWS; i++) {
    Result partialResult = null;
    Result completeResult = null;
    Result oneShotResult = null;
    partials.clear();

    do {
      partialResult = partialScanner.next();
      partials.add(partialResult);
    } while (partialResult != null && partialResult.mayHaveMoreCellsInRow());

    completeResult = Result.createCompleteResult(partials);
    oneShotResult = oneShotScanner.next();

    compareResults(completeResult, oneShotResult, null);
  }

  assertTrue(oneShotScanner.next() == null);
  assertTrue(partialScanner.next() == null);

  oneShotScanner.close();
  partialScanner.close();
}
 

@Test
public void testPartialResultWhenRegionMove() throws IOException {
  Table table = createTestTable(TableName.valueOf(name.getMethodName()),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.mayHaveMoreCellsInRow());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.mayHaveMoreCellsInRow());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.mayHaveMoreCellsInRow());

}
 

@Test
public void testReversedPartialResultWhenRegionMove() throws IOException {
  Table table = createTestTable(TableName.valueOf(name.getMethodName()),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  scan.setReversed(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS-1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.mayHaveMoreCellsInRow());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.mayHaveMoreCellsInRow());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.mayHaveMoreCellsInRow());

}
 

public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception {
  // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true
  // the results will NOT be reconstructed and instead the caller will see the partial results
  // returned by the server
  Scan partialScan = new Scan(basePartialScan);
  partialScan.setAllowPartialResults(true);
  ResultScanner partialScanner = TABLE.getScanner(partialScan);

  // Scan that retrieves all table results in single RPC request
  Scan oneShotScan = new Scan(basePartialScan);
  oneShotScan.setMaxResultSize(Long.MAX_VALUE);
  oneShotScan.setCaching(ROWS.length);
  ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);

  Result oneShotResult = oneShotScanner.next();
  Result partialResult = null;
  int iterationCount = 0;

  while (oneShotResult != null && oneShotResult.rawCells() != null) {
    List<Cell> aggregatePartialCells = new ArrayList<>();
    do {
      partialResult = partialScanner.next();
      assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null);
      assertTrue("Partial cells are null. iteration: " + iterationCount,
          partialResult.rawCells() != null);

      for (Cell c : partialResult.rawCells()) {
        aggregatePartialCells.add(c);
      }
    } while (partialResult.mayHaveMoreCellsInRow());

    assertTrue("Number of cells differs. iteration: " + iterationCount,
        oneShotResult.rawCells().length == aggregatePartialCells.size());
    final Cell[] oneShotCells = oneShotResult.rawCells();
    for (int cell = 0; cell < oneShotCells.length; cell++) {
      Cell oneShotCell = oneShotCells[cell];
      Cell partialCell = aggregatePartialCells.get(cell);

      assertTrue("One shot cell was null", oneShotCell != null);
      assertTrue("Partial cell was null", partialCell != null);
      assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell,
          oneShotCell.equals(partialCell));
    }

    oneShotResult = oneShotScanner.next();
    iterationCount++;
  }

  assertTrue(partialScanner.next() == null);

  partialScanner.close();
  oneShotScanner.close();
}