Java Code Examples for org.apache.hadoop.hbase.client.Table#getScanner()

protected void wipeOutMeta() throws IOException {
  // Mess it up by blowing up meta.
  Admin admin = TEST_UTIL.getAdmin();
  Scan s = new Scan();
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  ResultScanner scanner = meta.getScanner(s);
  List<Delete> dels = new ArrayList<>();
  for (Result r : scanner) {
    RegionInfo info =
        CatalogFamilyFormat.getRegionInfo(r);
    if(info != null && !info.getTable().getNamespaceAsString()
        .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
      Delete d = new Delete(r.getRow());
      dels.add(d);
      admin.unassign(r.getRow(), true);
    }
  }
  meta.delete(dels);
  scanner.close();
  meta.close();
}
 

@Test
public void testHBASE14489() throws IOException {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Table table = util.createTable(tableName, new byte[][] { A });
  Put put = new Put(ROW);
  put.addColumn(A, A, A);
  table.put(put);

  Scan s = new Scan();
  s.setFilter(new FilterAllFilter());
  ResultScanner scanner = table.getScanner(s);
  try {
    for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    }
  } finally {
    scanner.close();
  }
  verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
    tableName, new Boolean[] { true });
  util.deleteTable(tableName);
  table.close();

}
 

public static int getRowCount(Table table, boolean isRaw) throws IOException {
    Scan s = new Scan();
    s.setRaw(isRaw);;
    s.setMaxVersions();
    int rows = 0;
    try (ResultScanner scanner = table.getScanner(s)) {
        Result result = null;
        while ((result = scanner.next()) != null) {
            rows++;
            CellScanner cellScanner = result.cellScanner();
            Cell current = null;
            while (cellScanner.advance()) {
                current = cellScanner.current();
            }
        }
    }
    return rows;
}
 

private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store,
        ResultConverter converter) throws IOException {

    Table table = null;
    ResultScanner rs = null;
    Scan scan = new Scan();
    try {
        table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName));
        rs = table.getScanner(scan);
        converter.convertResult(rs, store);
        store.checkAndPutResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"),
                StringEntity.serializer);
    } finally {
        IOUtils.closeQuietly(rs);
        IOUtils.closeQuietly(table);
    }

}
 

private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 

public static CellCount getCellCount(Table table, boolean isRaw) throws IOException {
    Scan s = new Scan();
    s.setRaw(isRaw);;
    s.setMaxVersions();

    CellCount cellCount = new CellCount();
    try (ResultScanner scanner = table.getScanner(s)) {
        Result result = null;
        while ((result = scanner.next()) != null) {
            CellScanner cellScanner = result.cellScanner();
            Cell current = null;
            while (cellScanner.advance()) {
                current = cellScanner.current();
                cellCount.addCell(Bytes.toString(CellUtil.cloneRow(current)));
            }
        }
    }
    return cellCount;
}
 

private static void assertTenantIds(Expression e, Table htable, Filter filter, String[] tenantIds) throws IOException {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    Scan scan = new Scan();
    scan.setFilter(filter);
    ResultScanner scanner = htable.getScanner(scan);
    Result result = null;
    ResultTuple tuple;
    Set<String> actualTenantIds = Sets.newHashSetWithExpectedSize(tenantIds.length);
    Set<String> expectedTenantIds = new HashSet<>(Arrays.asList(tenantIds));
    while ((result = scanner.next()) != null) {
        tuple = new ResultTuple(result);
        e.evaluate(tuple, ptr);
        String tenantId = (String)PVarchar.INSTANCE.toObject(ptr);
        actualTenantIds.add(tenantId == null ? "" : tenantId);
    }
    assertTrue(actualTenantIds.containsAll(expectedTenantIds));
}
 

protected ResultScanner getResults(final Table table, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations) throws IOException {
    final Scan scan = new Scan();
    scan.setStartRow(startRow);
    scan.setStopRow(endRow);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (columns != null && columns.size() > 0) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 

private static int addToEachStartKey(final int expected) throws IOException {
  Table t = TEST_UTIL.getConnection().getTable(TABLENAME);
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    RegionInfo hri = CatalogFamilyFormat.getRegionInfo(r);
    if (hri == null) break;
    if (!hri.getTable().equals(TABLENAME)) {
      continue;
    }

    // If start key, add 'aaa'.
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.addColumn(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 

protected ResultScanner getResults(final Table table, final Collection<Column> columns, final Filter filter, final long minTime, List<String> authorizations) throws IOException {
    // Create a new scan. We will set the min timerange as the latest timestamp that
    // we have seen so far. The minimum timestamp is inclusive, so we will get duplicates.
    // We will record any cells that have the latest timestamp, so that when we scan again,
    // we know to throw away those duplicates.
    final Scan scan = new Scan();
    scan.setTimeRange(minTime, Long.MAX_VALUE);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (filter != null) {
        scan.setFilter(filter);
    }

    if (columns != null) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 

public void deleteAllRows(Connection conn, TableName tableName) throws SQLException,
    IOException {
    Scan scan = new Scan();
    Table table = conn.unwrap(PhoenixConnection.class).getQueryServices().
        getTable(tableName.getName());
    try (ResultScanner scanner = table.getScanner(scan)) {
        for (Result r : scanner) {
            Delete del = new Delete(r.getRow());
            table.delete(del);
        }
    }
}
 

private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
    throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes(cf));
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 

@VisibleForTesting
void deleteFromScan(Table stateTable, Scan scan) throws IOException {
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    Result next;
    while ((next = scanner.next()) != null) {
      stateTable.delete(new Delete(next.getRow()));
    }
  }
}
 

@Nullable
private TimeRegions getNextSetOfTimeRegions(Table stateTable, long time) throws IOException {
  byte[] timeBytes = Bytes.toBytes(getInvertedTime(time));
  Scan scan = new Scan(makeTimeRegionKey(timeBytes, EMPTY_BYTE_ARRAY), REGION_TIME_KEY_PREFIX_STOP);
  scan.addColumn(FAMILY, REGION_TIME_COL);


  long currentRegionTime = -1;
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  Result next;
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    while ((next = scanner.next()) != null) {
      Map.Entry<Long, byte[]> timeRegion = getTimeRegion(next.getRow());
      // Stop if reached next time value
      if (currentRegionTime == -1) {
        currentRegionTime = timeRegion.getKey();
      } else if (timeRegion.getKey() < currentRegionTime) {
        break;
      } else if (timeRegion.getKey() > currentRegionTime) {
        throw new IllegalStateException(
          String.format("Got out of order time %d when expecting time less than or equal to %d",
                        timeRegion.getKey(), currentRegionTime));
      }
      regions.add(timeRegion.getValue());
    }
  }
  return regions.isEmpty() ? null : new TimeRegions(currentRegionTime, Collections.unmodifiableSortedSet(regions));
}
 

@Test
public void testPartialResultWhenRegionMove() throws IOException {
  Table table = createTestTable(TableName.valueOf(name.getMethodName()),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.mayHaveMoreCellsInRow());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.mayHaveMoreCellsInRow());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.mayHaveMoreCellsInRow());

}
 

@SuppressWarnings("deprecation")
private int getKeyValueCount(Table table) throws IOException {
    Scan scan = new Scan();
    scan.setMaxVersions(Integer.MAX_VALUE - 1);

    ResultScanner results = table.getScanner(scan);
    int count = 0;
    for (Result res : results) {
      count += res.listCells().size();
      LOGGER.debug(count + ") " + res);
    }
    results.close();

    return count;
  }
 

@VisibleForTesting
void deleteFromScan(Table stateTable, Scan scan) throws IOException {
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    Result next;
    while ((next = scanner.next()) != null) {
      stateTable.delete(new Delete(next.getRow()));
    }
  }
}
 

/**
 * Run a dry run over the configured input, printing SolrInputDocuments to
 * the configured output.
 * 
 * @return 0 if successful, non-zero otherwise
 */
int run() {
  
    long programStartTime = System.currentTimeMillis();
    IndexingSpecification indexingSpec = indexingOpts.getIndexingSpecification();

    IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexingSpec.getIndexerComponentFactory(), new ByteArrayInputStream(indexingSpec.getConfiguration()), Maps.<String, String>newHashMap());
    IndexerConf indexerConf = factory.createIndexerConf();

    if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
        LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
        indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
    }

    Map<String, String> params = indexerConf.getGlobalParams();
    if (indexingOpts.morphlineFile != null) {
        params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
    }
    if (indexingOpts.morphlineId != null) {
        params.put(MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, indexingOpts.morphlineId);
    }

    for (Map.Entry<String, String> entry : hbaseConf) {
        if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) {
            params.put(entry.getKey(), entry.getValue());
        }
        if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) {
            params.put(entry.getKey(), entry.getValue());
        }
    }

    MorphlineClasspathUtil.setupJavaCompilerClasspath();
    
    ResultToSolrMapper resultToSolrMapper = ResultToSolrMapperFactory.createResultToSolrMapper(
            indexingSpec.getIndexerName(),
            indexerConf);
    
    Indexer indexer = Indexer.createIndexer(
                            indexingSpec.getIndexerName(),
                            indexerConf,
                            indexingSpec.getTableName(),
                            resultToSolrMapper,
                            null,
                            null,
                            documentWriter);
    
    Scan scan = indexingOpts.getScans().get(0);
    
    Table htable = null;
    Connection connection = null;
    try {
        connection = ConnectionFactory.createConnection(hbaseConf);
        htable = connection.getTable(TableName.valueOf(indexingSpec.getTableName()));
        ResultScanner scanner = htable.getScanner(scan);
        for (Result result : scanner) {
            indexer.indexRowData(ImmutableList.<RowData>of(new ResultWrappingRowData(result,
                    indexingSpec.getTableName().getBytes(Charsets.UTF_8))));
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        Closer.close(htable);
        if (connection != null) {
            try {
                connection.close();
            } catch (IOException ioe) {
                ; // ignore
            }
        }
    }
    
    HBaseMapReduceIndexerTool.goodbye(null, programStartTime);
    return 0;
    
}
 

@Override
public int run(String[] args) throws Exception {
  Options options = new Options();
  options.addOption("s", "start", true, "start key");
  options.addOption("e", "end", true, "end key");
  options.addOption("l", "limit", true, "number to print");

  GnuParser parser = new GnuParser();
  CommandLine cmd = null;
  try {
    cmd = parser.parse(options, args);
    if (cmd.getArgs().length != 0) {
      throw new ParseException("Command takes no arguments");
    }
  } catch (ParseException e) {
    System.err.println("Failed to parse command line " + e.getMessage());
    System.err.println();
    HelpFormatter formatter = new HelpFormatter();
    formatter.printHelp(getClass().getSimpleName(), options);
    System.exit(-1);
  }

  Connection connection = ConnectionFactory.createConnection(getConf());
  Table table = connection.getTable(getTableName(getConf()));

  Scan scan = new Scan();
  scan.setBatch(10000);

  if (cmd.hasOption("s"))
    scan.withStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));

  if (cmd.hasOption("e")) {
    scan.withStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
  }

  int limit = 0;
  if (cmd.hasOption("l"))
    limit = Integer.parseInt(cmd.getOptionValue("l"));
  else
    limit = 100;

  ResultScanner scanner = table.getScanner(scan);

  CINode node = new CINode();
  Result result = scanner.next();
  int count = 0;
  while (result != null && count++ < limit) {
    node = getCINode(result, node);
    System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
        Bytes.toStringBinary(node.prev), node.count, node.client);
    result = scanner.next();
  }
  scanner.close();
  table.close();
  connection.close();

  return 0;
}
 

@Test
public void testFilterWrapper() {
  int kv_number = 0;
  int row_number = 0;
  try {
    Scan scan = new Scan();
    List<Filter> fs = new ArrayList<>();

    DependentColumnFilter f1 = new DependentColumnFilter(Bytes.toBytes("f1"),
        Bytes.toBytes("c5"), true, CompareOperator.EQUAL,
        new SubstringComparator("c5"));
    PageFilter f2 = new PageFilter(2);
    fs.add(f1);
    fs.add(f2);
    FilterList filter = new FilterList(fs);

    scan.setFilter(filter);
    Table table = connection.getTable(name);
    ResultScanner scanner = table.getScanner(scan);

    // row2 (c1-c4) and row3(c1-c4) are returned
    for (Result result : scanner) {
      row_number++;
      for (Cell kv : result.listCells()) {
        LOG.debug(kv_number + ". kv: " + kv);
        kv_number++;
        assertEquals("Returned row is not correct", Bytes.toString(CellUtil.cloneRow(kv)),
            "row" + ( row_number + 1 ));
      }
    }

    scanner.close();
    table.close();
  } catch (Exception e) {
    // no correct result is expected
    assertNull("Exception happens in scan", e);
  }
  LOG.debug("check the fetched kv number");
  assertEquals("We should get 8 results returned.", 8, kv_number);
  assertEquals("We should get 2 rows returned", 2, row_number);
}