org.apache.hadoop.hbase.filter.FilterList#addFilter

Source File: ActiveMemberRunner.java From BigDataArchitect with Apache License 2.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList filterList = new FilterList();
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_MEMBER_ID, // 会员id
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION, // 浏览器版本号
            EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME // 添加一个事件名称获取列，在使用singlecolumnvaluefilter的时候必须指定对应的列是一个返回列
    };
    filterList.addFilter(this.getColumnFilter(columns));
    // 只需要page view事件，所以进行过滤
    filterList.addFilter(new SingleColumnValueFilter(ActiveMemberMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.PAGEVIEW.alias)));

    return filterList;
}

Source File: DefaultHBaseSerde.java From envelope with Apache License 2.0

6 votes

private FilterList getColumnValueFilters(Row row) {
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
  Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames());
  
  for (Map.Entry<String, ColumnDef> column : columns.entrySet()) {
    if (!column.getValue().cf.equals("rowkey")) {
      if (filterColumnNames.contains(column.getKey())) {
        byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row);
        if (value != null) {
          SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter(
              Bytes.toBytes(column.getValue().cf),
              Bytes.toBytes(column.getValue().name),
              CompareFilter.CompareOp.EQUAL,
              value
          );
          filterList.addFilter(columnValueFilter);
        }
      }
    }
  }
  
  return filterList;
}

Source File: RegionLocationCleaner.java From hbase-tools with Apache License 2.0

6 votes

@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}

Source File: HBaseReader.java From geowave with Apache License 2.0

6 votes

private void addSkipFilter(final RangeReaderParams<T> params, final FilterList filterList) {
  // Add skipping filter if requested
  if (params.getMaxResolutionSubsamplingPerDimension() != null) {
    if (params.getMaxResolutionSubsamplingPerDimension().length != params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length) {
      LOGGER.warn(
          "Unable to subsample for table '"
              + params.getIndex().getName()
              + "'. Subsample dimensions = "
              + params.getMaxResolutionSubsamplingPerDimension().length
              + " when indexed dimensions = "
              + params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length);
    } else {
      final int cardinalityToSubsample =
          IndexUtils.getBitPositionFromSubsamplingArray(
              params.getIndex().getIndexStrategy(),
              params.getMaxResolutionSubsamplingPerDimension());

      final FixedCardinalitySkippingFilter skippingFilter =
          new FixedCardinalitySkippingFilter(cardinalityToSubsample);
      filterList.addFilter(skippingFilter);
    }
  }
}

Source File: LocationRunner.java From BigDataPlatform with GNU General Public License v3.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
  //过滤数据，只分析launch事件
  FilterList filterList = new FilterList();
  filterList.addFilter(
      new SingleColumnValueExcludeFilter(Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME),
          Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL,
          Bytes.toBytes(
              EventEnum.PAGEVIEW.alias)));
  String[] columns = new String[]{
      EventLogConstants.LOG_COLUMN_NAME_SESSION_ID, EventLogConstants.LOG_COLUMN_NAME_UUID,
      EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, EventLogConstants.LOG_COLUMN_NAME_PLATFORM,
      EventLogConstants.LOG_COLUMN_NAME_COUNTRY,
      EventLogConstants.LOG_COLUMN_NAME_PROVINCE,
      EventLogConstants.LOG_COLUMN_NAME_CITY,
      EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME
  };
  filterList.addFilter(this.getColumnFilter(columns));
  return filterList;
}

Source File: TestThriftConnection.java From hbase with Apache License 2.0

6 votes

private void testScanWithFilters(Connection connection, String tableName) throws IOException {
  createTable(thriftAdmin, tableName);
  try (Table table = connection.getTable(TableName.valueOf(tableName))){
    FilterList filterList = new FilterList();
    PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow"));
    ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1,
        CompareOperator.EQUAL, VALUE_1);
    filterList.addFilter(prefixFilter);
    filterList.addFilter(columnValueFilter);
    Scan scan = new Scan();
    scan.readVersions(2);
    scan.setFilter(filterList);
    ResultScanner scanner = table.getScanner(scan);
    Iterator<Result> iterator = scanner.iterator();
    assertTrue(iterator.hasNext());
    int counter = 0;
    while (iterator.hasNext()) {
      Result result = iterator.next();
      counter += result.size();
    }
    assertEquals(2, counter);
  }
}

Source File: HBaseDemo.java From BigDataArchitect with Apache License 2.0

6 votes

/**
 * 查询某个用户所有的主叫电话（type=1）
 *  某个用户
 *  type=1
 *
 */
@Test
public void getType() throws IOException {
    Scan scan = new Scan();
    //创建过滤器集合
    FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    //创建过滤器
    SingleColumnValueFilter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"),Bytes.toBytes("type"),CompareOperator.EQUAL,Bytes.toBytes("1"));
    filters.addFilter(filter1);
    //前缀过滤器
    PrefixFilter filter2 = new PrefixFilter(Bytes.toBytes("15883348450"));
    filters.addFilter(filter2);
    scan.setFilter(filters);
    ResultScanner rss = table.getScanner(scan);
    for (Result rs:rss) {
        System.out.print(Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("dnum")))));
        System.out.print("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("type")))));
        System.out.print("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("length")))));
        System.out.println("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("date")))));
    }
}

Source File: VertexIndexModel.java From hgraphdb with Apache License 2.0

6 votes

private Scan getVertexIndexScanWithLimit(String label, boolean isUnique, String key, Object from, int limit, boolean reversed) {
    byte[] prefix = serializeForRead(label, isUnique, key, null);
    byte[] startRow = from != null
            ? serializeForRead(label, isUnique, key, from)
            : prefix;
    byte[] stopRow = HConstants.EMPTY_END_ROW;
    if (graph.configuration().getInstanceType() == HBaseGraphConfiguration.InstanceType.BIGTABLE) {
        if (reversed) {
            throw new UnsupportedOperationException("Reverse scans not supported by Bigtable");
        } else {
            // PrefixFilter in Bigtable does not automatically stop
            // See https://github.com/GoogleCloudPlatform/cloud-bigtable-client/issues/1087
            stopRow = HBaseGraphUtils.incrementBytes(prefix);
        }
    }
    if (reversed) startRow = HBaseGraphUtils.incrementBytes(startRow);
    Scan scan = new Scan(startRow, stopRow);
    FilterList filterList = new FilterList();
    filterList.addFilter(new PrefixFilter(prefix));
    filterList.addFilter(new PageFilter(limit));
    scan.setFilter(filterList);
    scan.setReversed(reversed);
    return scan;
}

Source File: CommandAdapter.java From hbase-tools with Apache License 2.0

6 votes

public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}

Source File: JobHistoryService.java From hraven with Apache License 2.0

6 votes

/**
 * creates a scan for flow data
 * @param rowPrefix - start row prefix
 * @param limit - limit on scanned results
 * @param version - version to match
 * @return Scan
 */
private Scan createFlowScan(byte[] rowPrefix, int limit, String version) {
  Scan scan = new Scan();
  scan.setStartRow(rowPrefix);

  // using a large scanner caching value with a small limit can mean we scan a
  // lot more data than necessary, so lower the caching for low limits
  scan.setCaching(Math.min(limit, defaultScannerCaching));
  // require that all rows match the prefix we're looking for
  Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix));
  // if version is passed, restrict the rows returned to that version
  if (version != null && version.length() > 0) {
    FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    filters.addFilter(prefixFilter);
    filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES,
        Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL,
        Bytes.toBytes(version)));
    scan.setFilter(filters);
  } else {
    scan.setFilter(prefixFilter);
  }
  return scan;
}

Source File: PageViewRunner.java From BigDataArchitect with Apache License 2.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList filterList = new FilterList();
    // 只需要pageview事件
    filterList.addFilter(new SingleColumnValueFilter(PageViewMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventLogConstants.EventEnum.PAGEVIEW.alias)));
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, // 获取事件名称
            EventLogConstants.LOG_COLUMN_NAME_CURRENT_URL, // 当前url
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本号
    };
    filterList.addFilter(this.getColumnFilter(columns));

    return filterList;
}

Source File: HGet.java From spliceengine with GNU Affero General Public License v3.0

6 votes

@Override
public void setFilter(DataFilter txnFilter){
    Filter toAdd;
    Filter existingFilter=get.getFilter();
    if(existingFilter!=null){
        FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        fl.addFilter(existingFilter);
        if(txnFilter instanceof HFilterWrapper)
            fl.addFilter(((HFilterWrapper)txnFilter).unwrapDelegate());
        else
            fl.addFilter(new HDataFilterWrapper(txnFilter));
        toAdd = fl;
    }else{
        if(txnFilter instanceof HFilterWrapper)
            toAdd = ((HFilterWrapper)txnFilter).unwrapDelegate();
        else
            toAdd = new HDataFilterWrapper(txnFilter);
    }
    get.setFilter(toAdd);
}

Source File: HbaseTraceDaoV2.java From pinpoint with Apache License 2.0

5 votes

private Filter getFilter(Filter filter1, Filter filter2) {
    if (filter1 != null && filter2 != null) {
        FilterList filterList = new FilterList();
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);
        return filterList;
    }
    if (filter1 != null) {
        return filter1;
    }
    if (filter2 != null) {
        return filter2;
    }
    return null;
}

Source File: Filters.java From java-docs-samples with Apache License 2.0

5 votes

public static void filterComposingInterleave(
    String projectId, String instanceId, String tableId) {
  // A filter that matches cells with the value true OR with the column qualifier os_build
  Filter qualifierFilter =
      new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("os_build")));
  Filter valueFilter =
      new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("true")));

  FilterList filter = new FilterList(Operator.MUST_PASS_ONE);
  filter.addFilter(qualifierFilter);
  filter.addFilter(valueFilter);

  Scan scan = new Scan().setFilter(filter).setMaxVersions();
  readWithFilter(projectId, instanceId, tableId, scan);
}

Source File: HBaseResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private FilterList generateTimeFilterList(VisitFilter visitFilter) {
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    if (visitFilter.lastModStart >= 0) { // NOTE: Negative value does not work in its binary form
        SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS,
                CompareFilter.CompareOp.GREATER_OR_EQUAL, Bytes.toBytes(visitFilter.lastModStart));
        filterList.addFilter(timeStartFilter);
    }
    if (visitFilter.lastModEndExclusive != Long.MAX_VALUE) {
        SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS,
                CompareFilter.CompareOp.LESS, Bytes.toBytes(visitFilter.lastModEndExclusive));
        filterList.addFilter(timeEndFilter);
    }
    return filterList.getFilters().isEmpty() ? null : filterList;
}

Source File: HBaseLogReader.java From eagle with Apache License 2.0

5 votes

/**
 * TODO If the required field is null for a row, then this row will not be fetched. That could be a
 * problem for counting Need another version of read to strictly get the number of rows which will return
 * all the columns for a column family
 */
@Override
public void open() throws IOException {
    if (isOpen) {
        return; // silently return
    }
    try {
        tbl = EagleConfigFactory.load().getHTable(schema.getTable());
    } catch (RuntimeException ex) {
        throw new IOException(ex);
    }

    String rowkeyRegex = buildRegex2(searchTags);
    RegexStringComparator regexStringComparator = new RegexStringComparator(rowkeyRegex);
    regexStringComparator.setCharset(Charset.forName("ISO-8859-1"));
    RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator);
    FilterList filterList = new FilterList();
    filterList.addFilter(filter);
    Scan s1 = new Scan();
    // reverse timestamp, startRow is stopKey, and stopRow is startKey
    s1.setStartRow(stopKey);
    s1.setStopRow(startKey);
    s1.setFilter(filterList);
    // TODO the # of cached rows should be minimum of (pagesize and 100)
    s1.setCaching(100);
    // TODO not optimized for all applications
    s1.setCacheBlocks(true);
    // scan specified columnfamily and qualifiers
    for (byte[] qualifier : qualifiers) {
        s1.addColumn(schema.getColumnFamily().getBytes(), qualifier);
    }
    rs = tbl.getScanner(s1);
    isOpen = true;
}

Source File: InboundRunner.java From BigDataArchitect with Apache License 2.0

5 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList list = new FilterList();
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_REFERRER_URL, // 前一个页面的url
            EventLogConstants.LOG_COLUMN_NAME_UUID, // uuid
            EventLogConstants.LOG_COLUMN_NAME_SESSION_ID, // 会话id
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME // 事件名称
    };
    list.addFilter(this.getColumnFilter(columns));
    list.addFilter(new SingleColumnValueFilter(InboundMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.PAGEVIEW.alias)));
    return list;
}

Source File: NewMemberRunner.java From BigDataPlatform with GNU General Public License v3.0

5 votes

@Override
protected Filter fetchHbaseFilter() {
  FilterList filterList = new FilterList();
  // 定义mapper中需要获取的列名
  String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_MEMBER_ID, // 会员id
      EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
      EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
      EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
      EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本信息
  };
  filterList.addFilter(this.getColumnFilter(columns));
  return filterList;
}

Source File: HBaseLogReader.java From Eagle with Apache License 2.0

5 votes

/**
 * TODO If the required field is null for a row, then this row will not be fetched. That could be a problem for counting
 * Need another version of read to strictly get the number of rows which will return all the columns for a column family
 */
public void open() throws IOException {
	if (isOpen)
		return; // silently return
	try {
		tbl = EagleConfigFactory.load().getHTable(schema.getTable());
	} catch (RuntimeException ex) {
		throw new IOException(ex);
	}

	String rowkeyRegex = buildRegex2(searchTags);
	RegexStringComparator regexStringComparator = new RegexStringComparator(
			rowkeyRegex);
	regexStringComparator.setCharset(Charset.forName("ISO-8859-1"));
	RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator);
	FilterList filterList = new FilterList();
	filterList.addFilter(filter);
	Scan s1 = new Scan();
	// reverse timestamp, startRow is stopKey, and stopRow is startKey
	s1.setStartRow(stopKey);
	s1.setStopRow(startKey);
	s1.setFilter(filterList);
	// TODO the # of cached rows should be minimum of (pagesize and 100)
	s1.setCaching(100);
	// TODO not optimized for all applications
	s1.setCacheBlocks(true);
	// scan specified columnfamily and qualifiers
	for(byte[] qualifier : qualifiers){
		s1.addColumn(schema.getColumnFamily().getBytes(), qualifier);
	}
	rs = tbl.getScanner(s1);
	isOpen = true;
}

Source File: QuotaTableUtil.java From hbase with Apache License 2.0

4 votes

/**
 * converts quotafilter to serializeable filterlists.
 */
public static Filter makeFilter(final QuotaFilter filter) {
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  if (StringUtils.isNotEmpty(filter.getUserFilter())) {
    FilterList userFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    boolean hasFilter = false;

    if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) {
      FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
      nsFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
      nsFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL,
          new RegexStringComparator(
            getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0)));
      userFilters.addFilter(nsFilters);
      hasFilter = true;
    }
    if (StringUtils.isNotEmpty(filter.getTableFilter())) {
      FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
      tableFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
      tableFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL,
          new RegexStringComparator(
            getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0)));
      userFilters.addFilter(tableFilters);
      hasFilter = true;
    }
    if (!hasFilter) {
      userFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
    }

    filterList.addFilter(userFilters);
  } else if (StringUtils.isNotEmpty(filter.getTableFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL,
        new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0)));
  } else if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL,
        new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0)));
  } else if (StringUtils.isNotEmpty(filter.getRegionServerFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(
        getRegionServerRowKeyRegex(filter.getRegionServerFilter()), 0)));
  }
  return filterList;
}

Java Code Examples for org.apache.hadoop.hbase.filter.FilterList#addFilter()