Java Code Examples for org.apache.accumulo.core.client.BatchScanner#iterator()

@Override
public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception {
    if (!genericConfig.getClass().getName().equals(ContentQueryConfiguration.class.getName())) {
        throw new QueryException("Did not receive a ContentQueryConfiguration instance!!");
    }
    
    final ContentQueryConfiguration config = (ContentQueryConfiguration) genericConfig;
    
    try {
        final BatchScanner scanner = this.scannerFactory
                        .newScanner(config.getTableName(), config.getAuthorizations(), this.queryThreads, config.getQuery());
        scanner.setRanges(config.getRanges());
        
        if (null != this.viewName) {
            final IteratorSetting cfg = new IteratorSetting(50, RegExFilter.class);
            cfg.addOption(RegExFilter.COLQ_REGEX, this.viewName);
            scanner.addScanIterator(cfg);
        }
        
        this.iterator = scanner.iterator();
        this.scanner = scanner;
        
    } catch (TableNotFoundException e) {
        throw new RuntimeException("Table not found: " + this.getTableName(), e);
    }
}
 

@Override
public boolean hasNext() {
    
    if (null == iterator) {
        try {
            BatchScanner scanner = connector.createBatchScanner(queryTable, connector.securityOperations().getUserAuthorizations(username), 100);
            
            scanner.setRanges(ranges);
            for (String cf : cfs) {
                
                scanner.fetchColumnFamily(new Text(cf));
                
            }
            
            iterator = scanner.iterator();
            
        } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
            throw new RuntimeException(e);
        }
    }
    nextIterator();
    return null != topKey;
}
 

public PCJKeyToJoinBindingSetIterator(BatchScanner scanner,
		Map<String, String> pcjVarMap,
		Map<String, Value> constantConstraints, int maxPrefixLen) {
	Preconditions.checkNotNull(scanner);
	Preconditions.checkArgument(pcjVarMap.size() > 0,
			"Variable map must contain at least one variable!");
	Preconditions.checkNotNull(constantConstraints,
			"Constant constraints cannot be null.");
	Preconditions.checkArgument(maxPrefixLen > 0,
			"Max prefix length must be greater than 0.");
	Preconditions
			.checkArgument(maxPrefixLen <= pcjVarMap.size(),
					"Max prefix length must be less than total number of binding names.");
	this.scanner = scanner;
	this.pcjVarMap = HashBiMap.create(pcjVarMap).inverse();
	this.constantConstraints = constantConstraints;
	this.maxPrefixLen = maxPrefixLen;
	this.iterator = scanner.iterator();

}
 

/**
 * Get the date type description which includes the fields and the mapped date range.
 * 
 * @param dateType
 * @param begin
 * @param end
 * @param datatypeFilter
 * @return the date type description
 * @throws TableNotFoundException
 */
@Cacheable(value = "getTypeDescription", key = "{#root.target.dateIndexTableName,#root.target.auths,#dateType,#begin,#end,#datatypeFilter}",
                cacheManager = "dateIndexHelperCacheManager")
public DateTypeDescription getTypeDescription(String dateType, Date begin, Date end, Set<String> datatypeFilter) throws TableNotFoundException {
    log.debug("cache fault for getTypeDescription(" + dateIndexTableName + ", " + auths + ", " + dateType + ", " + begin + ", " + end + ", "
                    + datatypeFilter + ")");
    if (log.isTraceEnabled()) {
        this.showMeDaCache("before getTypeDescription");
    }
    long startTime = System.currentTimeMillis();
    
    DateTypeDescription desc = new DateTypeDescription();
    
    BatchScanner bs = ScannerHelper.createBatchScanner(connector, dateIndexTableName, auths, numQueryThreads);
    try {
        
        // scan from begin to end
        bs.setRanges(Arrays.asList(new Range(DateIndexUtil.format(begin), DateIndexUtil.format(end) + '~')));
        
        // restrict to our date type
        bs.fetchColumnFamily(new Text(dateType));
        
        Iterator<Entry<Key,Value>> iterator = bs.iterator();
        
        while (iterator.hasNext()) {
            Entry<Key,Value> entry = iterator.next();
            Key k = entry.getKey();
            
            String[] parts = StringUtils.split(k.getColumnQualifier().toString(), '\0');
            if (datatypeFilter == null || datatypeFilter.isEmpty() || datatypeFilter.contains(parts[1])) {
                desc.fields.add(parts[2]);
                String date = parts[0];
                if (desc.dateRange[0] == null) {
                    desc.dateRange[0] = date;
                    desc.dateRange[1] = date;
                } else {
                    if (date.compareTo(desc.dateRange[0]) < 0) {
                        desc.dateRange[0] = date;
                    }
                    if (date.compareTo(desc.dateRange[1]) > 0) {
                        desc.dateRange[1] = date;
                    }
                }
            }
        }
    } finally {
        bs.close();
    }
    
    // if the dates are still empty, then default to the incoming dates
    if (desc.dateRange[0] == null) {
        desc.dateRange[0] = DateIndexUtil.format(begin);
        desc.dateRange[1] = DateIndexUtil.format(end);
    }
    
    if (log.isDebugEnabled()) {
        long endTime = System.currentTimeMillis();
        log.debug("getTypeDescription from table: " + dateIndexTableName + ", " + auths + ", " + dateType + ", " + begin + ", " + end + ", "
                        + datatypeFilter + " returned " + desc + " in " + (endTime - startTime) + "ms");
    }
    
    return desc;
}
 

@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    config = (EdgeQueryConfiguration) configuration;
    prefilterValues = null;
    EdgeQueryConfiguration.dateType dateFilterType = ((EdgeQueryConfiguration) configuration).getDateRangeType();
    
    log.debug("Performing edge table query: " + config.getQueryString());
    
    boolean includeStats = ((EdgeQueryConfiguration) configuration).includeStats();
    
    String queryString = config.getQueryString();
    String normalizedQuery = null;
    String statsNormalizedQuery = null;
    
    queryString = fixQueryString(queryString);
    QueryData qData = configureRanges(queryString);
    setRanges(qData.getRanges());
    
    VisitationContext context = null;
    try {
        context = normalizeJexlQuery(queryString, false);
        normalizedQuery = context.getNormalizedQuery().toString();
        statsNormalizedQuery = context.getNormalizedStatsQuery().toString();
        log.debug("Jexl after normalizing SOURCE and SINK: " + normalizedQuery);
    } catch (JexlException ex) {
        log.error("Error parsing user query.", ex);
    }
    
    if ((null == normalizedQuery || normalizedQuery.equals("")) && qData.getRanges().size() < 1) {
        throw new IllegalStateException("Query string is empty after initial processing, no ranges or filters can be generated to execute.");
    }
    
    addIterators(qData, getDateBasedIterators(config.getBeginDate(), config.getEndDate(), currentIteratorPriority, dateFilterSkipLimit, dateFilterType));
    
    if (!normalizedQuery.equals("")) {
        log.debug("Query being sent to the filter iterator: " + normalizedQuery);
        IteratorSetting edgeIteratorSetting = new IteratorSetting(currentIteratorPriority, EdgeFilterIterator.class.getSimpleName() + "_"
                        + currentIteratorPriority, EdgeFilterIterator.class);
        edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_OPTION, normalizedQuery);
        edgeIteratorSetting.addOption(EdgeFilterIterator.PROTOBUF_OPTION, "TRUE");
        
        if (!statsNormalizedQuery.equals("")) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.JEXL_STATS_OPTION, statsNormalizedQuery);
        }
        if (prefilterValues != null) {
            String value = serializePrefilter();
            edgeIteratorSetting.addOption(EdgeFilterIterator.PREFILTER_WHITELIST, value);
        }
        
        if (includeStats) {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "TRUE");
        } else {
            edgeIteratorSetting.addOption(EdgeFilterIterator.INCLUDE_STATS_OPTION, "FALSE");
        }
        
        addIterator(qData, edgeIteratorSetting);
    }
    
    log.debug("Configuring connection: tableName: " + config.getTableName() + ", auths: " + config.getAuthorizations());
    
    BatchScanner scanner = createBatchScanner(config);
    
    log.debug("Using the following ranges: " + qData.getRanges());
    
    if (context != null && context.isHasAllCompleteColumnFamilies()) {
        for (Text columnFamily : context.getColumnFamilies()) {
            scanner.fetchColumnFamily(columnFamily);
        }
        
    }
    
    scanner.setRanges(qData.getRanges());
    
    addCustomFilters(qData, currentIteratorPriority);
    
    for (IteratorSetting setting : qData.getSettings()) {
        scanner.addScanIterator(setting);
    }
    
    this.scanner = scanner;
    iterator = scanner.iterator();
}
 

private Iterator<String> getPredicates() {
    
    String auths = conf.get(ConfigUtils.CLOUDBASE_AUTHS);
    BatchScanner bs = null;
    try {
        bs = conn.createBatchScanner(tablePrefix + "prospects", new Authorizations(auths), 10);
    } catch (TableNotFoundException e) {
        e.printStackTrace();
        throw new Error("Attempting to scan missing table: " + tablePrefix + "prospects", e);
    }
    bs.setRanges(Collections.singleton(Range.prefix(new Text("predicate" + "\u0000"))));
    final Iterator<Entry<Key,Value>> iter = bs.iterator();
    
    return new Iterator<String>() {

        private String next = null;
        private boolean hasNextCalled = false;
        private boolean isEmpty = false;

        @Override
        public boolean hasNext() {

            if (!hasNextCalled && !isEmpty) {
                while (iter.hasNext()) {
                    Entry<Key,Value> temp = iter.next();
                    String row = temp.getKey().getRow().toString();
                    String[] rowArray = row.split("\u0000");
                    next = rowArray[1];
                    
                    hasNextCalled = true;
                    return true;
                }
                isEmpty = true;
                return false;
            } else if(isEmpty) {
                return false;
            }else {
                return true;
            }
        }

        @Override
        public String next() {

            if (hasNextCalled) {
                hasNextCalled = false;
                return next;
            } else if(isEmpty) {
                throw new NoSuchElementException();
            }else {
                if (this.hasNext()) {
                    hasNextCalled = false;
                    return next;
                } else {
                    throw new NoSuchElementException();
                }
            }
        }

        @Override
        public void remove() {

            throw new UnsupportedOperationException("Cannot delete from iterator!");

        }

    }; 
}
 

@Override
public void execute(final OperationParams params) throws ParseException {
  final StopWatch stopWatch = new StopWatch();

  // Ensure we have all the required arguments
  if (parameters.size() != 1) {
    throw new ParameterException("Requires arguments: <storename>");
  }

  final String storeName = parameters.get(0);

  // Attempt to load store.
  final StoreLoader storeOptions = new StoreLoader(storeName);
  if (!storeOptions.loadFromConfig(getGeoWaveConfigFile(params), params.getConsole())) {
    throw new ParameterException("Cannot find store name: " + storeOptions.getStoreName());
  }

  final String storeType = storeOptions.getDataStorePlugin().getType();

  if (storeType.equals(AccumuloStoreFactoryFamily.TYPE)) {
    try {
      final AccumuloRequiredOptions opts =
          (AccumuloRequiredOptions) storeOptions.getFactoryOptions();

      final AccumuloOperations ops =
          new AccumuloOperations(
              opts.getZookeeper(),
              opts.getInstance(),
              opts.getUser(),
              opts.getPassword(),
              opts.getGeoWaveNamespace(),
              (AccumuloOptions) opts.getStoreOptions());

      long results = 0;
      final BatchScanner scanner = ops.createBatchScanner(indexId);
      scanner.setRanges(Collections.singleton(new Range()));
      final Iterator<Entry<Key, Value>> it = scanner.iterator();

      stopWatch.start();
      while (it.hasNext()) {
        it.next();
        results++;
      }
      stopWatch.stop();

      scanner.close();
      System.out.println("Got " + results + " results in " + stopWatch.toString());
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
      LOGGER.error("Unable to scan accumulo datastore", e);
    }
  } else if (storeType.equals(HBaseStoreFactoryFamily.TYPE)) {
    throw new UnsupportedOperationException(
        "full scan for store type " + storeType + " not yet implemented.");
  } else {
    throw new UnsupportedOperationException(
        "full scan for store type " + storeType + " not implemented.");
  }
}