Java Code Examples for org.apache.hadoop.hbase.client.Scan#setFilter()

@Test
public void testReadVersionWithRawAndFilter() throws IOException {
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
          , CellComparator.getInstance(), false);
  KeyValue [] kvs = new KeyValue[] {
          create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[]{
      new KeyValueScanFixture(CellComparator.getInstance(), kvs)
    });

  BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
  Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
  scanSpec.setFilter(filter);
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(2, results.size());
  }
}
 

public static void andFilterAtBeginning(Scan scan, Filter andWithFilter) {
    if (andWithFilter == null) {
        return;
    }
    Filter filter = scan.getFilter();
    if (filter == null) {
        scan.setFilter(andWithFilter); 
    } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
        FilterList filterList = (FilterList)filter;
        List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
        allFilters.add(andWithFilter);
        allFilters.addAll(filterList.getFilters());
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
    } else {
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(andWithFilter, filter)));
    }
}
 

/**
 * Returns the {@link Flow} instance containing the given job ID.
 *
 * @param cluster the cluster identifier
 * @param jobId the job identifier
 * @return
 */
public Flow getFlowByJobID(String cluster, String jobId,
    boolean populateTasks) throws IOException {
  Flow flow = null;
  JobKey key = idService.getJobKeyById(new QualifiedJobId(cluster, jobId));
  if (key != null) {
    byte[] startRow =
        ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(key.getCluster()),
            Bytes.toBytes(key.getUserName()), Bytes.toBytes(key.getAppId()),
            Bytes.toBytes(key.getEncodedRunId()), Constants.EMPTY_BYTES);

    LOG.info("Reading job_history rows start at "
        + Bytes.toStringBinary(startRow));
    Scan scan = new Scan();
    // start scanning history at cluster!user!app!run!
    scan.setStartRow(startRow);
    // require that all results match this flow prefix
    scan.setFilter(new WhileMatchFilter(new PrefixFilter(startRow)));

    List<Flow> flows = createFromResults(scan, populateTasks, 1);
    if (flows.size() > 0) {
      flow = flows.get(0);
    }
  }
  return flow;
}
 

@Override
protected void initialize(JobContext job) throws IOException {
  Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(
      job.getConfiguration()));
  TableName tableName = TableName.valueOf("exampleTable");
  // mandatory
  initializeTable(connection, tableName);
  byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
    Bytes.toBytes("columnB") };
  //optional
  Scan scan = new Scan();
  for (byte[] family : inputColumns) {
    scan.addFamily(family);
  }
  Filter exampleFilter =
    new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*"));
  scan.setFilter(exampleFilter);
  setScan(scan);
}
 

@Override
public void configure(JobConf job) {
  try {
    Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job));
    TableName tableName = TableName.valueOf("exampleJobConfigurableTable");
    // mandatory
    initializeTable(connection, tableName);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    //optional
    Scan scan = new Scan();
    for (byte[] family : inputColumns) {
      scan.addFamily(family);
    }
    Filter exampleFilter =
      new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*"));
    scan.setFilter(exampleFilter);
    setScan(scan);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to initialize.", exception);
  }
}
 

public static void andFilterAtEnd(Scan scan, Filter andWithFilter) {
    if (andWithFilter == null) {
        return;
    }
    Filter filter = scan.getFilter();
    if (filter == null) {
        scan.setFilter(andWithFilter); 
    } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
        FilterList filterList = (FilterList)filter;
        List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
        allFilters.addAll(filterList.getFilters());
        allFilters.add(andWithFilter);
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
    } else {
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(filter, andWithFilter)));
    }
}
 

/**
 * Returns a count of the rows in the region where this coprocessor is loaded.
 */
@Override
public void getRowCount(RpcController controller, CountRequest request,
                        RpcCallback<CountResponse> done) {
  Scan scan = new Scan();
  scan.setFilter(new FirstKeyOnlyFilter());
  CountResponse response = null;
  InternalScanner scanner = null;
  try {
    scanner = env.getRegion().getScanner(scan);
    List<Cell> results = new ArrayList<>();
    boolean hasMore = false;
    byte[] lastRow = null;
    long count = 0;
    do {
      hasMore = scanner.next(results);
      for (Cell kv : results) {
        byte[] currentRow = CellUtil.cloneRow(kv);
        if (lastRow == null || !Bytes.equals(lastRow, currentRow)) {
          lastRow = currentRow;
          count++;
        }
      }
      results.clear();
    } while (hasMore);

    response = CountResponse.newBuilder()
        .setCount(count).build();
  } catch (IOException ioe) {
    CoprocessorRpcUtils.setControllerException(controller, ioe);
  } finally {
    if (scanner != null) {
      try {
        scanner.close();
      } catch (IOException ignored) {}
    }
  }
  done.run(response);
}
 

private Scan buildScan() {
  final Scan metaScan = new Scan()
    .addFamily(HConstants.CATALOG_FAMILY)
    .readVersions(1)
    .setLimit((scanLimit != null ? scanLimit : SCAN_LIMIT_DEFAULT) + 1);
  if (scanStart != null) {
    metaScan.withStartRow(scanStart, false);
  }
  final Filter filter = buildScanFilter();
  if (filter != null) {
    metaScan.setFilter(filter);
  }
  return metaScan;
}
 

private ScanModifier newScanModifier(final String value) {
  return new ScanModifier() {
    @Override
    public Scan modifyScan(Scan scan) {
      scan.addColumn(Bytes.toBytes("meta"), Bytes.toBytes("testcf"));
      SingleColumnValueFilter filter = new SingleColumnValueFilter(
          Bytes.toBytes("meta"), Bytes.toBytes("testcf"),
          CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value));
      filter.setFilterIfMissing(true);
      scan.setFilter(filter);
      return scan;
    }
  };
}
 

@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
  throws IOException {
  Transaction tx = getFromOperation(scan);
  if (tx != null) {
    projectFamilyDeletes(scan);
    scan.setMaxVersions();
    scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
                      TxUtils.getMaxVisibleTimestamp(tx));
    Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
    scan.setFilter(newFilter);
  }
  return s;
}
 

protected Scan getPropertyScan(String label, byte[] key, byte[] inclusiveFromValue, byte[] exclusiveToValue) {
    Scan scan = new Scan();
    SingleColumnValueFilter labelFilter = new SingleColumnValueFilter(Constants.DEFAULT_FAMILY_BYTES,
            Constants.LABEL_BYTES, CompareFilter.CompareOp.EQUAL, new BinaryComparator(ValueUtils.serialize(label)));
    labelFilter.setFilterIfMissing(true);
    SingleColumnValueFilter fromValueFilter = new SingleColumnValueFilter(Constants.DEFAULT_FAMILY_BYTES,
            key, CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator(inclusiveFromValue));
    fromValueFilter.setFilterIfMissing(true);
    SingleColumnValueFilter toValueFilter = new SingleColumnValueFilter(Constants.DEFAULT_FAMILY_BYTES,
            key, CompareFilter.CompareOp.LESS, new BinaryComparator(exclusiveToValue));
    toValueFilter.setFilterIfMissing(true);
    FilterList filterList = new FilterList(labelFilter, fromValueFilter, toValueFilter);
    scan.setFilter(filterList);
    return scan;
}
 

/**
 * Retrieves all the event rows matching a single
 * {@link com.twitter.hraven.Flow}.
 * @param flowKey
 * @return
 */
public List<FlowEvent> getFlowEvents(FlowKey flowKey) throws IOException {
  byte[] startKey =
      Bytes.add(flowKeyConverter.toBytes(flowKey), Constants.SEP_BYTES);
  Scan scan = new Scan(startKey);
  scan.setFilter(new WhileMatchFilter(new PrefixFilter(startKey)));

  List<FlowEvent> results = new ArrayList<FlowEvent>();
  ResultScanner scanner = null;
  Table eventTable = null;
  try {
    eventTable = hbaseConnection
        .getTable(TableName.valueOf(Constants.FLOW_EVENT_TABLE));
    scanner = eventTable.getScanner(scan);
    for (Result r : scanner) {
      FlowEvent event = createEventFromResult(r);
      if (event != null) {
        results.add(event);
      }
    }
  } finally {
    try {
      if (scanner != null) {
        scanner.close();
      }
    } finally {
      if (eventTable != null) {
        eventTable.close();
      }
    }
  }
  return results;
}
 

@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s)
  throws IOException {
  Transaction tx = getFromOperation(scan);
  if (tx != null) {
    projectFamilyDeletes(scan);
    scan.setMaxVersions();
    scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData),
                      TxUtils.getMaxVisibleTimestamp(tx));
    Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter());
    scan.setFilter(newFilter);
  }
  return s;
}
 

@Test
public void testSkipFilter() throws IOException {

  // Test for qualifier regex: "testQualifierOne-2"
  // Should only get rows from second group, and all keys
  Filter f = new SkipFilter(new QualifierFilter(CompareOperator.NOT_EQUAL,
      new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))));
  Scan s = new Scan();
  s.setFilter(f);

  KeyValue [] kvs = {
      // testRowTwo-0
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
      // testRowTwo-2
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
      // testRowTwo-3
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
  };
  verifyScanFull(s, kvs);
}
 

@Test
public void testReverseMultiRowRangeFilterIncludingMinAndMaxRow() throws IOException {
  tableName = TableName.valueOf(name.getMethodName());
  Table ht = TEST_UTIL.createTable(tableName, family);
  for (String rowkey : Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h")) {
    byte[] row = Bytes.toBytes(rowkey);
    Put p = new Put(row);
    p.addColumn(family, qf, value);
    ht.put(p);
  }
  TEST_UTIL.flush();

  Scan scan = new Scan();
  scan.setReversed(true);
  List<RowRange> ranges = Arrays.asList(
      new RowRange(Bytes.toBytes("a"), true, Bytes.toBytes("c"), true),
      new RowRange(Bytes.toBytes("f"), true, Bytes.toBytes("h"), true)
  );
  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);

  List<String> expected = Arrays.asList("h", "g", "f", "c", "b", "a");
  List<String> actual = new ArrayList<>();
  for (Cell cell : getResults(ht, scan)) {
    actual.add(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
  }

  assertEquals(expected, actual);
}
 

/**
 * Test the filter by adding all columns of family A and B in the scan. (KO: row '3' without
 * 'a:foo' qualifier is returned)
 */
@Test
public void scanWithAllQualifiersOfBothFamilies() throws IOException {
  /* When */
  Scan scan = new Scan();
  scan.setFilter(scanFilter);

  verify(scan);
}
 

@Override
public void batchStarted(MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
    // The entire purpose of this method impl is to get the existing rows for the
    // table rows being indexed into the block cache, as the index maintenance code
    // does a point scan per row
    List<KeyRange> keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size());
    Map<ImmutableBytesWritable, IndexMaintainer> maintainers =
            new HashMap<ImmutableBytesWritable, IndexMaintainer>();
    ImmutableBytesWritable indexTableName = new ImmutableBytesWritable();
    for (int i = 0; i < miniBatchOp.size(); i++) {
        Mutation m = miniBatchOp.getOperation(i);
        keys.add(PVarbinary.INSTANCE.getKeyRange(m.getRow()));
        List<IndexMaintainer> indexMaintainers = getCodec().getIndexMaintainers(m.getAttributesMap());
        
        for(IndexMaintainer indexMaintainer: indexMaintainers) {
            if (indexMaintainer.isImmutableRows() && indexMaintainer.isLocalIndex()) continue;
            indexTableName.set(indexMaintainer.getIndexTableName());
            if (maintainers.get(indexTableName) != null) continue;
            maintainers.put(indexTableName, indexMaintainer);
        }
        
    }
    if (maintainers.isEmpty()) return;
    Scan scan = IndexManagementUtil.newLocalStateScan(new ArrayList<IndexMaintainer>(maintainers.values()));
    ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN);
    scanRanges.initializeScan(scan);
    scan.setFilter(scanRanges.getSkipScanFilter());
    HRegion region = this.env.getRegion();
    RegionScanner scanner = region.getScanner(scan);
    // Run through the scanner using internal nextRaw method
    region.startRegionOperation();
    try {
        boolean hasMore;
        do {
            List<Cell> results = Lists.newArrayList();
            // Results are potentially returned even when the return value of s.next is false
            // since this is an indication of whether or not there are more values after the
            // ones returned
            hasMore = scanner.nextRaw(results);
        } while (hasMore);
    } finally {
        try {
            scanner.close();
        } finally {
            region.closeRegionOperation();
        }
    }
}
 

private static void scanMeta(Connection connection, @Nullable final byte[] startRow,
  @Nullable final byte[] stopRow, QueryType type, @Nullable Filter filter, int maxRows,
  final ClientMetaTableAccessor.Visitor visitor) throws IOException {
  int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE;
  Scan scan = getMetaScan(connection, rowUpperLimit);

  for (byte[] family : type.getFamilies()) {
    scan.addFamily(family);
  }
  if (startRow != null) {
    scan.withStartRow(startRow);
  }
  if (stopRow != null) {
    scan.withStopRow(stopRow);
  }
  if (filter != null) {
    scan.setFilter(filter);
  }

  if (LOG.isTraceEnabled()) {
    LOG.trace("Scanning META" + " starting at row=" + Bytes.toStringBinary(startRow) +
      " stopping at row=" + Bytes.toStringBinary(stopRow) + " for max=" + rowUpperLimit +
      " with caching=" + scan.getCaching());
  }

  int currentRow = 0;
  try (Table metaTable = getMetaHTable(connection)) {
    try (ResultScanner scanner = metaTable.getScanner(scan)) {
      Result data;
      while ((data = scanner.next()) != null) {
        if (data.isEmpty()) {
          continue;
        }
        // Break if visit returns false.
        if (!visitor.visit(data)) {
          break;
        }
        if (++currentRow >= rowUpperLimit) {
          break;
        }
      }
    }
  }
  if (visitor instanceof Closeable) {
    try {
      ((Closeable) visitor).close();
    } catch (Throwable t) {
      ExceptionUtil.rethrowIfInterrupt(t);
      LOG.debug("Got exception in closing the meta scanner visitor", t);
    }
  }
}
 

@Test
public void testCreateDropTable() throws Exception {
    long ts = nextTimestamp();
    String tenantId = getOrganizationId();
    initATableValues(tenantId, getDefaultSplits(tenantId), null, ts);
    
    ensureTableCreated(getUrl(), BTABLE_NAME, null, ts-2);
    ensureTableCreated(getUrl(), PTSDB_NAME, null, ts-2);
    
    Properties props = new Properties();
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 5));
    Connection conn5 = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
    String query = "SELECT a_string FROM aTable";
    // Data should still be there b/c we only dropped the schema
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 8));
    assertTrue(conn5.prepareStatement(query).executeQuery().next());
    conn5.createStatement().executeUpdate("DROP TABLE " + ATABLE_NAME);
    
    // Confirm that data is no longer there because we dropped the table
    // This needs to be done natively b/c the metadata is gone
    HTableInterface htable = conn5.unwrap(PhoenixConnection.class).getQueryServices().getTable(SchemaUtil.getTableNameAsBytes(ATABLE_SCHEMA_NAME, ATABLE_NAME));
    Scan scan = new Scan();
    scan.setFilter(new FirstKeyOnlyFilter());
    scan.setTimeRange(0, ts+9);
    assertNull(htable.getScanner(scan).next());
    conn5.close();

    // Still should work b/c we're at an earlier timestamp than when table was deleted
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2));
    Connection conn2 = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
    assertTrue(conn2.prepareStatement(query).executeQuery().next());
    conn2.close();
    
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
    Connection conn10 = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
    try {
        conn10.prepareStatement(query).executeQuery().next();
        fail();
    } catch (TableNotFoundException e) {
    }
}
 

protected void initRecordScanner() {
  final FilterList filterList = new FilterList();
  final ByteArrayRange range = SplitsProvider.fromRowRange(recordReaderParams.getRowRange());

  final Scan rscanner = scanProvider.get();

  // TODO all datastores that use the default splitsprovider seem to
  // ignore range.isEndInclusive()
  // and use next prefix for the end of the scan range - this seems likely
  // to be overly inclusive, but doesn't seem to produce extra results for
  // the other datastores within GeoWaveBasicSparkIT, however it does for
  // HBase
  rscanner.setStartRow(range.getStart()).setStopRow(range.getEndAsNextPrefix());

  if (operations.isServerSideLibraryEnabled()) {
    addSkipFilter((RangeReaderParams<T>) recordReaderParams, filterList);
  }

  setLimit(recordReaderParams, filterList);
  if (!filterList.getFilters().isEmpty()) {
    if (filterList.getFilters().size() > 1) {
      rscanner.setFilter(filterList);
    } else {
      rscanner.setFilter(filterList.getFilters().get(0));
    }
  }

  Iterable<Result> resultScanner;
  try {
    resultScanner =
        operations.getScannedResults(rscanner, recordReaderParams.getIndex().getName());
    if (resultScanner instanceof ResultScanner) {
      this.scanner = (Closeable) resultScanner;
    }
  } catch (final IOException e) {
    LOGGER.error("Could not get the results from scanner", e);
    this.scanner = null;
    this.scanIt = null;
    return;
  }

  this.scanIt =
      this.rowTransformer.apply(
          Iterators.transform(
              resultScanner.iterator(),
              e -> new HBaseRow(e, partitionKeyLength)));
}