Java Code Examples for org.apache.accumulo.core.client.BatchScanner#addScanIterator()

The following examples show how to use org.apache.accumulo.core.client.BatchScanner#addScanIterator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AccumuloEventStore.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
@Override
public CloseableIterable<Event> query(Date start, Date end, Set<String> types, Node node, Set<String> selectFields, Auths auths) {
    checkNotNull(start);
    checkNotNull(end);
    checkNotNull(types);
    checkNotNull(node);
    checkNotNull(auths);

    BatchScanner indexScanner = helper.buildIndexScanner(auths.getAuths());
    GlobalIndexVisitor globalIndexVisitor = new EventGlobalIndexVisitor(start, end, types, indexScanner, shardBuilder);
    BatchScanner scanner = helper.buildShardScanner(auths.getAuths());

    IteratorSetting timeFilter = new IteratorSetting(5, EventTimeLimitingFilter.class);
    EventTimeLimitingFilter.setCurrentTime(timeFilter, end.getTime());
    EventTimeLimitingFilter.setTTL(timeFilter, end.getTime() - start.getTime());
    scanner.addScanIterator(timeFilter);

    CloseableIterable<Event> events = helper.query(scanner, globalIndexVisitor, types, node, helper.buildQueryXform(), selectFields, auths);
    indexScanner.close();

    return events;
}
 
Example 2
Source File: ContentQueryTable.java    From datawave with Apache License 2.0 6 votes vote down vote up
@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
    if (!genericConfig.getClass().getName().equals(ContentQueryConfiguration.class.getName())) {
        throw new QueryException("Did not receive a ContentQueryConfiguration instance!!");
    }
    
    final ContentQueryConfiguration config = (ContentQueryConfiguration) genericConfig;
    
    try {
        final BatchScanner scanner = this.scannerFactory
                        .newScanner(config.getTableName(), config.getAuthorizations(), this.queryThreads, config.getQuery());
        scanner.setRanges(config.getRanges());
        
        if (null != this.viewName) {
            final IteratorSetting cfg = new IteratorSetting(50, RegExFilter.class);
            cfg.addOption(RegExFilter.COLQ_REGEX, this.viewName);
            scanner.addScanIterator(cfg);
        }
        
        this.iterator = scanner.iterator();
        this.scanner = scanner;
        
    } catch (TableNotFoundException e) {
        throw new RuntimeException("Table not found: " + this.getTableName(), e);
    }
}
 
Example 3
Source File: ShardQueryLogic.java    From datawave with Apache License 2.0 6 votes vote down vote up
public static BatchScanner createBatchScanner(ShardQueryConfiguration config, ScannerFactory scannerFactory, QueryData qd) throws TableNotFoundException {
    final BatchScanner bs = scannerFactory.newScanner(config.getShardTableName(), config.getAuthorizations(), config.getNumQueryThreads(),
                    config.getQuery());
    
    if (log.isTraceEnabled()) {
        log.trace("Running with " + config.getAuthorizations() + " and " + config.getNumQueryThreads() + " threads: " + qd);
    }
    
    bs.setRanges(qd.getRanges());
    
    for (IteratorSetting cfg : qd.getSettings()) {
        bs.addScanIterator(cfg);
    }
    
    return bs;
}
 
Example 4
Source File: AccumuloFeatureStore.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<String> names(String group, String type, String prefix, Auths auths) {

    checkNotNull(group);
    checkNotNull(type);
    checkNotNull(prefix);
    checkNotNull(auths);

    try {
        BatchScanner scanner = connector.createBatchScanner(tableName + INDEX_SUFFIX, auths.getAuths(), config.getMaxQueryThreads());
        scanner.setRanges(singleton(Range.prefix(group + NULL_BYTE + type + NULL_BYTE + prefix)));

        IteratorSetting setting = new IteratorSetting(25, FirstEntryInRowIterator.class);
        scanner.addScanIterator(setting);

        return transform(wrap(scanner), groupTypeNameIndexTransform);
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    }
}
 
Example 5
Source File: AccumuloFeatureStore.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<String> types(String group, String prefix, Auths auths) {

    checkNotNull(group);
    checkNotNull(prefix);
    checkNotNull(auths);

    try {
        BatchScanner scanner = connector.createBatchScanner(tableName + INDEX_SUFFIX, auths.getAuths(), config.getMaxQueryThreads());
        scanner.setRanges(singleton(Range.prefix(group + NULL_BYTE + prefix)));

        IteratorSetting setting = new IteratorSetting(25, TypeIndexIterator.class);
        scanner.addScanIterator(setting);

        return transform(wrap(scanner), typeIndexTransform);
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    }
}
 
Example 6
Source File: AccumuloFeatureStore.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<String> groups(String prefix, Auths auths) {

    checkNotNull(prefix);
    checkNotNull(auths);
    try {
        BatchScanner scanner = connector.createBatchScanner(tableName + INDEX_SUFFIX, auths.getAuths(), config.getMaxQueryThreads());
        scanner.setRanges(singleton(Range.prefix(prefix)));

        IteratorSetting setting = new IteratorSetting(25, GroupIndexIterator.class);
        scanner.addScanIterator(setting);

        return transform(wrap(scanner), groupIndexTransform);
    } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
    }
}
 
Example 7
Source File: KeyValueIndex.java    From accumulo-recipes with Apache License 2.0 6 votes vote down vote up
/**
 * Returns all the unique event types in the event store.
 * @param auths
 * @return
 */
public CloseableIterable<String> getTypes(String prefix, Auths auths) {

    checkNotNull(auths);

    try {
        BatchScanner scanner = connector.createBatchScanner(indexTable, auths.getAuths(), config.getMaxQueryThreads());
        IteratorSetting setting = new IteratorSetting(15, GlobalIndexTypesIterator.class);

        scanner.addScanIterator(setting);
        scanner.setRanges(singletonList(new Range(INDEX_T + INDEX_SEP + prefix, INDEX_T + INDEX_SEP + prefix + "\uffff")));

        return transform(closeableIterable(scanner), new Function<Map.Entry<Key, Value>, String>() {
            @Override
            public String apply(Map.Entry<Key, Value> keyValueEntry) {
                String[] parts = splitByWholeSeparatorPreserveAllTokens(keyValueEntry.getKey().getRow().toString(), INDEX_SEP);
                return splitPreserveAllTokens(parts[1], NULL_BYTE)[0];
            }
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 8
Source File: DiscoveryLogic.java    From datawave with Apache License 2.0 5 votes vote down vote up
public static BatchScanner configureBatchScannerForDiscovery(DiscoveryQueryConfiguration config, ScannerFactory scannerFactory, String tableName,
                Collection<Range> seekRanges, Set<Text> columnFamilies, Multimap<String,String> literals, Multimap<String,String> patterns,
                Multimap<String,LiteralRange<String>> ranges, boolean reverseIndex) throws TableNotFoundException {
    
    // if we have no ranges, then nothing to scan
    if (seekRanges.isEmpty()) {
        return null;
    }
    
    BatchScanner bs = scannerFactory.newScanner(tableName, config.getAuthorizations(), config.getNumQueryThreads(), config.getQuery());
    bs.setRanges(seekRanges);
    if (!columnFamilies.isEmpty()) {
        for (Text family : columnFamilies) {
            bs.fetchColumnFamily(family);
        }
    }
    
    // The begin date from the query may be down to the second, for doing lookups in the index we want to use the day because
    // the times in the index table have been truncated to the day.
    Date begin = DateUtils.truncate(config.getBeginDate(), Calendar.DAY_OF_MONTH);
    // we don't need to bump up the end date any more because it's not apart of the range set on the scanner
    Date end = config.getEndDate();
    
    LongRange dateRange = new LongRange(begin.getTime(), end.getTime());
    
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDateRangeFilter(config, bs, dateRange);
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDataTypeFilter(config, bs, config.getDatatypeFilter());
    
    configureIndexMatchingIterator(config, bs, literals, patterns, ranges, reverseIndex);
    
    IteratorSetting discoveryIteratorSetting = new IteratorSetting(config.getBaseIteratorPriority() + 50, DiscoveryIterator.class);
    discoveryIteratorSetting.addOption(REVERSE_INDEX, Boolean.toString(reverseIndex));
    discoveryIteratorSetting.addOption(SEPARATE_COUNTS_BY_COLVIS, config.getSeparateCountsByColVis().toString());
    if (config.getShowReferenceCount()) {
        discoveryIteratorSetting.addOption(SHOW_REFERENCE_COUNT, config.getShowReferenceCount().toString());
    }
    bs.addScanIterator(discoveryIteratorSetting);
    
    return bs;
}
 
Example 9
Source File: AccumuloEventStore.java    From accumulo-recipes with Apache License 2.0 5 votes vote down vote up
/**
 * This method will batch get a bunch of events by uuid (and optionally timestamp). If another store is used to
 * index into events in this store in a specially designed way (i.e. getting the last-n events, etc...) then
 * the most optimal way to store the index would be the UUID and the timestamp. However, if all that is known
 * about an event is the uuid, this method will do the extra fetch of the timestamp from the index table.
 */
@Override
public CloseableIterable<Event> get(Collection<EventIdentifier> uuids, Set<String> selectFields, Auths auths) {
    checkNotNull(uuids);
    checkNotNull(auths);
    try {

        BatchScanner eventScanner = helper.buildShardScanner(auths.getAuths());
        Collection<Range> eventRanges = new LinkedList<Range>();

        for (EventIdentifier curIndex : uuids) {
            String shardId = shardBuilder.buildShard(EventBuilder.create(curIndex.getType(), curIndex.getId(), curIndex.getTimestamp()).build());
            eventRanges.add(prefix(shardId, PREFIX_E + ONE_BYTE + curIndex.getType() + ONE_BYTE + curIndex.getId()));
        }

        eventScanner.setRanges(eventRanges);

        if (selectFields != null && selectFields.size() > 0) {
            IteratorSetting iteratorSetting = new IteratorSetting(25, SelectFieldsExtractor.class);
            SelectFieldsExtractor.setSelectFields(iteratorSetting, selectFields);
            eventScanner.addScanIterator(iteratorSetting);
        }

        IteratorSetting expirationFilter = new IteratorSetting(23, "metaExpiration", MetadataExpirationFilter.class);
        eventScanner.addScanIterator(expirationFilter);

        IteratorSetting setting = new IteratorSetting(26, WholeColumnFamilyIterator.class);
        eventScanner.addScanIterator(setting);

        return transform(closeableIterable(eventScanner), helper.buildWholeColFXform());
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 10
Source File: AccumuloEntityStore.java    From accumulo-recipes with Apache License 2.0 5 votes vote down vote up
@Override
public CloseableIterable<Entity> getAllByType(Set<String> types, Set<String> selectFields, Auths auths) {
    checkNotNull(types);
    checkNotNull(auths);
    try {

        BatchScanner scanner = helper.buildShardScanner(auths.getAuths());

        Collection<Range> ranges = new LinkedList<Range>();
        for (String type : types) {
            Set<Text> shards = shardBuilder.buildShardsForTypes(singleton(type));
            for (Text shard : shards)
                ranges.add(prefix(shard.toString(), PREFIX_E + ONE_BYTE + type));
        }

        scanner.setRanges(ranges);

        if (selectFields != null && selectFields.size() > 0) {
            IteratorSetting iteratorSetting = new IteratorSetting(16, SelectFieldsExtractor.class);
            SelectFieldsExtractor.setSelectFields(iteratorSetting, selectFields);
            scanner.addScanIterator(iteratorSetting);
        }


        IteratorSetting expirationFilter = new IteratorSetting(7, "metaExpiration", MetadataExpirationFilter.class);
        scanner.addScanIterator(expirationFilter);


        IteratorSetting setting = new IteratorSetting(18, WholeColumnFamilyIterator.class);
        scanner.addScanIterator(setting);


        return transform(closeableIterable(scanner), helper.buildWholeColFXform());
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 11
Source File: AccumuloEntityStore.java    From accumulo-recipes with Apache License 2.0 5 votes vote down vote up
@Override
public CloseableIterable<Entity> get(Collection<EntityIdentifier> typesAndIds, Set<String> selectFields, Auths auths) {
    checkNotNull(typesAndIds);
    checkNotNull(auths);
    try {

        BatchScanner scanner = helper.buildShardScanner(auths.getAuths());

        Collection<Range> ranges = new LinkedList<Range>();
        for (EntityIdentifier curIndex : typesAndIds) {
            String shardId = shardBuilder.buildShard(curIndex.getType(), curIndex.getId());
            ranges.add(exact(shardId, PREFIX_E + ONE_BYTE + curIndex.getType() + ONE_BYTE + curIndex.getId()));
        }

        scanner.setRanges(ranges);

        if (selectFields != null && selectFields.size() > 0) {
            IteratorSetting iteratorSetting = new IteratorSetting(16, SelectFieldsExtractor.class);
            SelectFieldsExtractor.setSelectFields(iteratorSetting, selectFields);
            scanner.addScanIterator(iteratorSetting);
        }

        IteratorSetting expirationFilter = new IteratorSetting(7, "metaExpiration", MetadataExpirationFilter.class);
        scanner.addScanIterator(expirationFilter);

        IteratorSetting setting = new IteratorSetting(18, WholeColumnFamilyIterator.class);
        scanner.addScanIterator(setting);


        return transform(closeableIterable(scanner), helper.buildWholeColFXform());
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 12
Source File: EdgeTableWrapper.java    From AccumuloGraph with Apache License 2.0 5 votes vote down vote up
public Iterable<Edge> getEdges(String key, Object value) {
  AccumuloGraphUtils.nullCheckProperty(key, value);
  if (key.equalsIgnoreCase("label")) {
    key = Constants.LABEL;
  }
  
  BatchScanner scan = getBatchScanner();
  scan.fetchColumnFamily(new Text(key));

  byte[] val = AccumuloByteSerializer.serialize(value);
  if (val[0] != AccumuloByteSerializer.SERIALIZABLE) {
    IteratorSetting is = new IteratorSetting(10, "filter", RegExFilter.class);
    RegExFilter.setRegexs(is, null, null, null, Pattern.quote(new String(val)), false);
    scan.addScanIterator(is);

    return new ScannerIterable<Edge>(scan) {

      @Override
      public Edge next(PeekingIterator<Entry<Key,Value>> iterator) {

        Key k = iterator.next().getKey();

        if (k.getColumnFamily().toString().equals(Constants.LABEL)) {
          String[] vals = k.getColumnQualifier().toString().split(Constants.ID_DELIM);
          return new AccumuloEdge(globals, k.getRow().toString(),
              new AccumuloVertex(globals, vals[0]),
              new AccumuloVertex(globals, vals[1]), null);
        }
        return new AccumuloEdge(globals, k.getRow().toString());
      }
    };
  } else {
    // TODO
    throw new UnsupportedOperationException("Filtering on binary data not currently supported.");
  }
}
 
Example 13
Source File: BaseTableCache.java    From datawave with Apache License 2.0 5 votes vote down vote up
public void setupScanner(BatchScanner scanner) {
    scanner.setRanges(Lists.newArrayList(new Range()));
    Map<String,String> options = new HashMap<>();
    options.put(RegExFilter.COLF_REGEX, "^f$");
    options.put("negate", "true");
    IteratorSetting settings = new IteratorSetting(100, "skipFColumn", RegExFilter.class, options);
    scanner.addScanIterator(settings);
}
 
Example 14
Source File: BulkInputFormat.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * If maxVersions has been set, configure a {@link VersioningIterator} at priority 0 for this scanner.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @param scanner
 *            the scanner to configure
 */
protected void setupMaxVersions(Configuration conf, BatchScanner scanner) {
    int maxVersions = getMaxVersions(conf);
    // Check to make sure its a legit value
    if (maxVersions >= 1) {
        IteratorSetting vers = new IteratorSetting(0, "vers", VersioningIterator.class);
        VersioningIterator.setMaxVersions(vers, maxVersions);
        scanner.addScanIterator(vers);
    }
}
 
Example 15
Source File: ShardIndexQueryTable.java    From datawave with Apache License 2.0 5 votes vote down vote up
public static BatchScanner configureBatchScannerForDiscovery(ShardQueryConfiguration config, ScannerFactory scannerFactory, String tableName,
                Collection<Range> ranges, Collection<String> literals, Collection<String> patterns, boolean reverseIndex, boolean uniqueTermsOnly,
                Collection<String> expansionFields) throws TableNotFoundException {
    
    // if we have no ranges, then nothing to scan
    if (ranges.isEmpty()) {
        return null;
    }
    
    BatchScanner bs = scannerFactory.newScanner(tableName, config.getAuthorizations(), config.getNumQueryThreads(), config.getQuery());
    
    bs.setRanges(ranges);
    
    // The begin date from the query may be down to the second, for doing lookups in the index we want to use the day because
    // the times in the index table have been truncated to the day.
    Date begin = DateUtils.truncate(config.getBeginDate(), Calendar.DAY_OF_MONTH);
    // we don't need to bump up the end date any more because it's not apart of the range set on the scanner
    Date end = config.getEndDate();
    
    LongRange dateRange = new LongRange(begin.getTime(), end.getTime());
    
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDateRangeFilter(config, bs, dateRange);
    ShardIndexQueryTableStaticMethods.configureGlobalIndexDataTypeFilter(config, bs, config.getDatatypeFilter());
    
    ShardIndexQueryTableStaticMethods.configureGlobalIndexTermMatchingIterator(config, bs, literals, patterns, reverseIndex, uniqueTermsOnly,
                    expansionFields);
    
    bs.addScanIterator(new IteratorSetting(config.getBaseIteratorPriority() + 50, DiscoveryIterator.class));
    
    return bs;
}
 
Example 16
Source File: QueryScannerHelper.java    From datawave with Apache License 2.0 5 votes vote down vote up
public static BatchScanner createBatchScanner(Connector connector, String tableName, Collection<Authorizations> authorizations, int numQueryThreads,
                Query query, boolean reportErrors) throws TableNotFoundException {
    BatchScanner batchScanner = ScannerHelper.createBatchScanner(connector, tableName, authorizations, numQueryThreads);
    
    batchScanner.addScanIterator(getQueryInfoIterator(query, reportErrors));
    
    return batchScanner;
}
 
Example 17
Source File: AccumuloEventStore.java    From accumulo-recipes with Apache License 2.0 4 votes vote down vote up
@Override
public CloseableIterable<Event> getAllByType(Date start, Date stop, Set<String> types, Set<String> selectFields, Auths auths) {
    checkNotNull(types);
    checkNotNull(auths);
    try {

        BatchScanner scanner = helper.buildShardScanner(auths.getAuths());

        BatchScanner typeShardScanner = helper.buildIndexScanner(auths.getAuths());

        Set<Range> typeIndexRanges = new HashSet<Range>();
        for(String type : types) {

            Key typeStartKey = new Key("t__" + type + NULL_BYTE + shardBuilder.buildShard(start.getTime(), 0));
            Key typeStopKey = new Key("t__" + type + NULL_BYTE + shardBuilder.buildShard(stop.getTime(), shardBuilder.numPartitions()));
            typeIndexRanges.add(new Range(typeStartKey, typeStopKey));
        }
        typeShardScanner.setRanges(typeIndexRanges);

        Collection<Range> ranges = new LinkedList<Range>();
        for(Entry<Key,Value> entry : typeShardScanner) {
            String[] parts = splitPreserveAllTokens(entry.getKey().getRow().toString(), NULL_BYTE);
            String[] typeParts = splitByWholeSeparatorPreserveAllTokens(parts[0], "__");
            ranges.add(prefix(parts[1], PREFIX_E + ONE_BYTE + typeParts[1] + ONE_BYTE));
        }

        scanner.setRanges(ranges);

        if (selectFields != null && selectFields.size() > 0) {
            IteratorSetting iteratorSetting = new IteratorSetting(16, SelectFieldsExtractor.class);
            SelectFieldsExtractor.setSelectFields(iteratorSetting, selectFields);
            scanner.addScanIterator(iteratorSetting);
        }

        IteratorSetting timeFilter = new IteratorSetting(12, EventTimeLimitingFilter.class);
        EventTimeLimitingFilter.setCurrentTime(timeFilter, stop.getTime());

        EventTimeLimitingFilter.setTTL(timeFilter, stop.getTime() - start.getTime());
        scanner.addScanIterator(timeFilter);

        IteratorSetting setting = new IteratorSetting(18, WholeColumnFamilyIterator.class);
        scanner.addScanIterator(setting);

        IteratorSetting expirationFilter = new IteratorSetting(13, "metaExpiration", MetadataExpirationFilter.class);
        scanner.addScanIterator(expirationFilter);

        return transform(closeableIterable(scanner), helper.buildWholeColFXform());
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 18
Source File: EdgeQueryLogic.java    From datawave with Apache License 2.0 4 votes vote down vote up
@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    config = (EdgeQueryConfiguration) configuration;
    prefilterValues = null;
    EdgeQueryConfiguration.dateType dateFilterType = ((EdgeQueryConfiguration) configuration).getDateRangeType();
    
    log.debug("Performing edge table query: " + config.getQueryString());
    
    boolean includeStats = ((EdgeQueryConfiguration) configuration).includeStats();
    
    String queryString = config.getQueryString();
    String normalizedQuery = null;
    String statsNormalizedQuery = null;
    
    queryString = fixQueryString(queryString);
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    
    VisitationContext context = null;
    try {
        context = normalizeJexlQuery(queryString, false);
        normalizedQuery = context.getNormalizedQuery().toString();
        statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
        log.debug("Jexl after normalizing SOURCE and SINK: " + normalizedQuery);
    } catch (JexlException ex) {
        log.error("Error parsing user query.", ex);
    }
    
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterType));
    
    if (!normalizedQuery.equals("")) {
        log.debug("Query being sent to the filter iterator: " + normalizedQuery);
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_"
                        + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        
        addIterator(qData, edgeIteratorSetting);
    }
    
    log.debug("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    
    BatchScanner scanner = createBatchScanner(config);
    
    log.debug("Using the following ranges: " + qData.getRanges());
    
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
        
    }
    
    scanner.setRanges(qData.getRanges());
    
    addCustomFilters(qData, currentIteratorPriority);
    
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    
    this.scanner = scanner;
    iterator = scanner.iterator();
}
 
Example 19
Source File: AccumuloTemporalLastNStore.java    From accumulo-recipes with Apache License 2.0 4 votes vote down vote up
@Override
public CloseableIterable<Event> get(Date start, Date stop, Set<String> groups, int n, Auths auths) {

    List<Iterable<Event>> cursors = new LinkedList<Iterable<Event>>();
    String stopDay = generateTimestamp(start.getTime(), TimeUnit.DAYS);
    String startDay = generateTimestamp(stop.getTime(), TimeUnit.DAYS);

    String stopMinute = generateTimestamp(start.getTime(), TimeUnit.MINUTES);
    String startMinute = generateTimestamp(stop.getTime(), TimeUnit.MINUTES);

    String stopMillis = encoder.encode(start.getTime());
    String startMillis = encoder.encode(stop.getTime());

    for (String group : groups) {

        Key startKey = new Key(group + GROUP_DELIM + startDay, startMinute, startMillis + ONE_BYTE);
        Key stopKey = new Key(group + GROUP_DELIM + stopDay, stopMinute, stopMillis + ONE_BYTE + END_BYTE);

        try {
            BatchScanner scanner = connector.createBatchScanner(tableName, auths.getAuths(), 1);
            scanner.setRanges(singletonList(new Range(startKey, stopKey)));

            IteratorSetting setting = new IteratorSetting(7, WholeColumnQualifierIterator.class);
            scanner.addScanIterator(setting);

            IteratorSetting setting2 = new IteratorSetting(15, FirstNEntriesInRowIterator.class);
            FirstNEntriesInRowIterator.setNumKeysToReturn(setting2, n);
            scanner.addScanIterator(setting2);

            for (Map.Entry<Key, Value> entry : scanner) {
                List<Map.Entry<Key, Value>> topEntries = decodeRow(entry.getKey(), entry.getValue());
                Iterable<List<Map.Entry<Key, Value>>> entries = transform(topEntries, rowDecodeXform);
                cursors.add(transform(entries, entryXform));
            }

            scanner.close();

        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    return wrap(new EventMergeJoinIterable(cursors));
}
 
Example 20
Source File: AccumuloRecordCursor.java    From presto with Apache License 2.0 4 votes vote down vote up
public AccumuloRecordCursor(
        AccumuloRowSerializer serializer,
        BatchScanner scanner,
        String rowIdName,
        List<AccumuloColumnHandle> columnHandles)
{
    this.columnHandles = requireNonNull(columnHandles, "columnHandles is null");
    this.scanner = requireNonNull(scanner, "scanner is null");
    this.serializer = requireNonNull(serializer, "serializer is null");
    this.serializer.setRowIdName(requireNonNull(rowIdName, "rowIdName is null"));

    requireNonNull(columnHandles, "columnHandles is null");

    if (retrieveOnlyRowIds(rowIdName)) {
        this.scanner.addScanIterator(new IteratorSetting(1, "firstentryiter", FirstEntryInRowIterator.class));

        fieldToColumnName = new String[1];
        fieldToColumnName[0] = rowIdName;

        // Set a flag on the serializer saying we are only going to be retrieving the row ID
        this.serializer.setRowOnly(true);
    }
    else {
        // Else, we will be scanning some more columns here
        this.serializer.setRowOnly(false);

        // Fetch the reserved row ID column
        this.scanner.fetchColumn(ROW_ID_COLUMN, ROW_ID_COLUMN);

        Text family = new Text();
        Text qualifier = new Text();

        // Create an array which maps the column ordinal to the name of the column
        fieldToColumnName = new String[columnHandles.size()];
        for (int i = 0; i < columnHandles.size(); ++i) {
            AccumuloColumnHandle columnHandle = columnHandles.get(i);
            fieldToColumnName[i] = columnHandle.getName();

            // Make sure to skip the row ID!
            if (!columnHandle.getName().equals(rowIdName)) {
                // Set the mapping of presto column name to the family/qualifier
                this.serializer.setMapping(columnHandle.getName(), columnHandle.getFamily().get(), columnHandle.getQualifier().get());

                // Set our scanner to fetch this family/qualifier column
                // This will help us prune which data we receive from Accumulo
                family.set(columnHandle.getFamily().get());
                qualifier.set(columnHandle.getQualifier().get());
                this.scanner.fetchColumn(family, qualifier);
            }
        }
    }

    IteratorSetting setting = new IteratorSetting(WHOLE_ROW_ITERATOR_PRIORITY, WholeRowIterator.class);
    scanner.addScanIterator(setting);

    iterator = this.scanner.iterator();
}