Java Code Examples for org.apache.accumulo.core.iterators.SortedKeyValueIterator#hasTop()

protected static void findTopEnhanced(
    final SortedKeyValueIterator<Key, Value> source,
    final Filter filter) {
  Key key;
  if (source.hasTop()) {
    key = source.getTopKey();
  } else {
    return;
  }
  while (!key.isDeleted() && !filter.accept(key, source.getTopValue())) {
    try {
      source.next();
      if (source.hasTop()) {
        key = source.getTopKey();
      } else {
        return;
      }
    } catch (final IOException e) {
      throw new RuntimeException(e);
    }
  }
}
 

public TestData(SortedKeyValueIterator<Key,Value> iter, Range range, boolean reseek) {
    try {
        iter.seek(range, new HashSet<>(), false);
        
        while (iter.hasTop()) {
            data.put(iter.getTopKey(), iter.getTopValue());
            if (reseek) {
                iter.seek(new Range(iter.getTopKey(), false, range.getEndKey(), range.isEndKeyInclusive()), new HashSet<>(), false);
            } else {
                iter.next();
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Advance an iterator until skip(...) returns false. May be a combination of seek() and next() calls
 * 
 * @param itr
 * @param pointer
 * @param currentRange
 * @param columnFamilies
 * @param includeColumnFamilies
 * @throws IOException
 */
protected void advanceItr(SortedKeyValueIterator<Key,Value> itr, ByteSequence pointer, Range currentRange, Collection<ByteSequence> columnFamilies,
                boolean includeColumnFamilies) throws IOException {
    Key current = itr.getTopKey();
    Text row = current.getRow();
    int nextCount = 0;
    while (current != null && skip(current, row, pointer)) {
        if (maxNextCount == -1 || nextCount < maxNextCount) {
            itr.next();
            nextCount++;
        } else {
            Key startKey = getSeekStartKey(current, pointer);
            Range newRange = new Range(startKey, false, currentRange.getEndKey(), currentRange.isEndKeyInclusive());
            itr.seek(newRange, columnFamilies, includeColumnFamilies);
            nextCount = 0;
        }
        
        current = itr.hasTop() ? itr.getTopKey() : null;
    }
}
 

/**
 * A bit of a hack, similar to the ColumnFamilySkippingIterator. This will call next a 32 times before finally seeking for the next {@code <row, colf>}.
 * 
 * The source iterator may or may not have a top after this method returns, and there is no guarantee of another viable top key/value being set.
 */
public static void moveTo(Key key, SortedKeyValueIterator<Key,Value> iterator, Range scanRange, Collection<ByteSequence> cfs, boolean inclusive)
                throws IOException {
    int nexts = 0;
    boolean movedEnough = false;
    if (log.isTraceEnabled()) {
        log.trace("Iterator key is " + key);
        log.trace("Iterator has top?" + iterator.hasTop());
        if (iterator.hasTop())
            log.trace("Iterator top key?" + iterator.getTopKey());
        log.trace("ScanRange is ?" + scanRange);
        log.trace("Scan cfs are " + cfs + " is inclusive ? " + inclusive);
    }
    while (nexts < 32 && iterator.hasTop() && (movedEnough = iterator.getTopKey().compareTo(key) < 0)) {
        iterator.next();
        nexts++;
    }
    if (log.isTraceEnabled()) {
        log.trace("Nexts is " + nexts);
        log.trace("Moved enough? " + movedEnough);
    }
    if (!movedEnough && iterator.hasTop()) {
        iterator.seek(new Range(key, true, scanRange.getEndKey(), scanRange.isEndKeyInclusive()), cfs, inclusive);
    }
}
 

private void runTest(boolean reseek, TreeMap<Key,Value> source, TreeMap<Key,Value> result,
    Collection<ByteSequence> cols) throws IOException {
  MapIterator src = new MapIterator(source);
  SortedKeyValueIterator<Key,Value> iter = new ChunkCombiner();
  iter.init(src, null, null);
  iter = iter.deepCopy(null);
  iter.seek(new Range(), cols, true);

  TreeMap<Key,Value> seen = new TreeMap<>();

  while (iter.hasTop()) {
    assertFalse("already contains " + iter.getTopKey(), seen.containsKey(iter.getTopKey()));
    seen.put(new Key(iter.getTopKey()), new Value(iter.getTopValue()));

    if (reseek)
      iter.seek(new Range(iter.getTopKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL), true,
          null, true), cols, true);
    else
      iter.next();
  }

  assertEquals(result, seen);
}
 

public Key apply(SortedKeyValueIterator<Key,Value> itr) throws IOException {
    Key key = itr.getTopKey();
    ByteSequence parentId = parsePointer(key);
    Text row = key.getRow();
    Key nextKey = key;
    do {
        key = nextKey;
        itr.next();
        nextKey = (itr.hasTop() ? itr.getTopKey() : null);
    } while (skip(nextKey, row, parentId));
    return getResult(key, parentId);
}
 

@Override
protected SortedKeyValueIterator<Key,Value> createIndexOnlyKey(ASTEQNode node) throws IOException {
    Key newStartKey = getKey(node);
    
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (null == op || null == op.getLiteralValue()) {
        // deep copy since this is likely a null literal
        return source.deepCopy(env);
    }
    
    String fn = op.deconstructIdentifier();
    String literal = String.valueOf(op.getLiteralValue());
    
    if (log.isTraceEnabled()) {
        log.trace("createIndexOnlyKey for " + fn + " " + literal + " " + newStartKey);
    }
    List<Entry<Key,Value>> kv = Lists.newArrayList();
    if (null != limitedMap.get(Maps.immutableEntry(fn, literal))) {
        kv.add(limitedMap.get(Maps.immutableEntry(fn, literal)));
    } else {
        
        SortedKeyValueIterator<Key,Value> mySource = limitedSource;
        // if source size > 0, we are free to use up to that number for this query
        if (source.getSourceSize() > 0)
            mySource = source.deepCopy(env);
        
        mySource.seek(new Range(newStartKey, true, newStartKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false), Collections.emptyList(), false);
        
        if (mySource.hasTop()) {
            kv.add(Maps.immutableEntry(mySource.getTopKey(), Constants.NULL_VALUE));
            
        }
    }
    
    return new IteratorToSortedKeyValueIterator(kv.iterator());
}
 

protected SortedKeyValueIterator<Key,Value> createIndexOnlyKey(ASTEQNode node) throws IOException {
    Key newStartKey = getKey(node);
    
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (null == op || null == op.getLiteralValue()) {
        // deep copy since this is likely a null literal
        return source.deepCopy(env);
    }
    
    String fn = op.deconstructIdentifier();
    String literal = String.valueOf(op.getLiteralValue());
    
    if (log.isTraceEnabled()) {
        log.trace("createIndexOnlyKey for " + fn + " " + literal + " " + newStartKey);
    }
    List<Entry<Key,Value>> kv = Lists.newArrayList();
    if (null != limitedMap.get(Maps.immutableEntry(fn, literal))) {
        kv.add(limitedMap.get(Maps.immutableEntry(fn, literal)));
    } else {
        
        SortedKeyValueIterator<Key,Value> mySource = limitedSource;
        // if source size > 0, we are free to use up to that number for this query
        if (source.getSourceSize() > 0)
            mySource = source.deepCopy(env);
        
        mySource.seek(new Range(newStartKey, true, newStartKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false), Collections.emptyList(), false);
        
        if (mySource.hasTop()) {
            kv.add(Maps.immutableEntry(mySource.getTopKey(), Constants.NULL_VALUE));
            
        }
    }
    
    return new IteratorToSortedKeyValueIterator(kv.iterator());
}
 

@Override
public Key apply(SortedKeyValueIterator<Key,Value> itr) throws IOException {
    Key key = itr.getTopKey();
    Text row = key.getRow();
    ByteSequence pointer = parsePointer(key.getColumnQualifierData());
    while (itr.hasTop() && samePointer(row, pointer, itr.getTopKey()))
        itr.next();
    
    return TLD.buildParentKey(row, pointer, parseFieldNameValue(key.getColumnFamilyData(), key.getColumnQualifierData()), key.getColumnVisibility(),
                    key.getTimestamp());
}
 

public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {

        // always start fresh
        this.setTopKey(null);
        this.setDone(false);

        // get my user object which should be an iterator
        SortedKeyValueIterator<?, ?> iter = (SortedKeyValueIterator<?, ?>) this.getUserObject();
        if (iter != null) {

            iter.seek(range, columnFamilies, inclusive);

            if (iter.hasTop()) {
                Key key = (Key) iter.getTopKey();
                key = buildKey(key);

                this.setTopKey(key);
                if (log.isDebugEnabled()) {
                    log.debug("BLTNODE.seek() -> found: " + this.getTopKey());
                }
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("BLTNODE.seek() -> hasTop::false");
                }
                this.setDone(true);
            }
        } else {
            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.seek(), The iterator was null!");
            }
            this.setTopKey(null);
        }
    }
 

@Test
public void dataIntegrity_alternatingTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // re-seek
    copy1.seek(new Range(), Collections.emptyList(), false);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // alternating next loops
    int alternatingCount = 0;
    while (copy1.hasTop() && copy2.hasTop()) {
        assertTrue(copy1.getTopKey().equals(copy2.getTopKey()));
        alternatingCount++;
        copy1.next();
        copy2.next();
    }
    
    assertFalse(copy1.hasTop());
    assertFalse(copy2.hasTop());
    assertEquals(26, alternatingCount);
}
 

@Test
public void dataIntegrity_emptyRangeTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // test one side empty range
    copy1.seek(new Range(new Key("20121126_9"), true, new Key("20121126_99"), false), Collections.emptyList(), false);
    copy2.seek(new Range(new Key("20121126_0"), true, new Key("20121126_4"), true), Collections.emptyList(), false);
    
    int mixedCopy1Count = 0;
    int mixedCopy2Count = 0;
    
    while (copy1.hasTop() || copy2.hasTop()) {
        if (copy1.hasTop()) {
            copy1.getTopKey();
            mixedCopy1Count++;
            copy1.next();
        }
        
        if (copy2.hasTop()) {
            copy2.getTopKey();
            mixedCopy2Count++;
            copy2.next();
        }
    }
    
    assertTrue(mixedCopy2Count > mixedCopy1Count);
    assertTrue(mixedCopy2Count == 26);
    assertTrue(mixedCopy1Count == 0);
}
 

public Map<Set<Tag>, List<Aggregation>> subquery(QueryRequest msg, QueryRequest.SubQuery query)
        throws TimelyException {

    Map<Set<Tag>, List<Aggregation>> aggregationList = new HashMap<>();
    long start = System.currentTimeMillis();
    try {
        SortedKeyValueIterator<org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value> itr = null;
        Collection<Authorizations> authorizations = getSessionAuthorizations(msg);
        itr = setupIterator(msg, query, authorizations, getAgeOffForMetric(query.getMetric()));
        Map<Set<Tag>, Set<Tag>> matchingTagCache = new HashMap<>();
        while (itr.hasTop()) {
            Map<Set<Tag>, Aggregation> samples = AggregationIterator.decodeValue(itr.getTopValue());
            for (Map.Entry<Set<Tag>, Aggregation> entry : samples.entrySet()) {
                Set<Tag> allMatchingTags = (Set<Tag>) matchingTagCache.get(entry.getKey());
                if (allMatchingTags == null) {
                    allMatchingTags = new HashSet<>();
                    for (Tag tag : entry.getKey()) {
                        if (query.getTags().keySet().contains(tag.getKey())) {
                            allMatchingTags.add(tag);
                        }
                    }
                    matchingTagCache.put(entry.getKey(), allMatchingTags);
                }
                List<Aggregation> aggregations = aggregationList.getOrDefault(allMatchingTags, new ArrayList<>());
                aggregations.add(entry.getValue());
                aggregationList.put(allMatchingTags, aggregations);
            }
            itr.next();
        }
        return aggregationList;
    } catch (Exception e) {
        LOG.error("Error during query: " + e.getMessage(), e);
        throw new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
                "Error during query: " + e.getMessage(), e.getMessage(), e);
    } finally {
        LOG.trace("Time for cache subquery for {} - {}ms", query.toString(), System.currentTimeMillis() - start);
    }
}
 

@Test
public void testRateIterator() throws TimelyException {

    long BASETIME = System.currentTimeMillis();
    // align basetime to a downsample period
    BASETIME = BASETIME - (BASETIME % 1000);
    DataStoreCache mmStore = getMetricMemoryStore2(BASETIME);

    QueryRequest query = new QueryRequest();
    query.setStart(BASETIME);
    query.setEnd(BASETIME + 1440000);
    query.setMsResolution(true);
    QueryRequest.SubQuery subQuery = new QueryRequest.SubQuery();
    subQuery.setDownsample(Optional.of("1ms-avg"));
    subQuery.setMetric("metric.number.1");
    subQuery.addTag("host", ".*");
    QueryRequest.RateOption rateOption = new QueryRequest.RateOption();
    rateOption.setCounter(false);
    subQuery.setRate(true);
    subQuery.setRateOptions(rateOption);
    query.setQueries(Collections.singleton(subQuery));

    SortedKeyValueIterator<org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value> itr = null;
    try {
        // long firstTimestamp = Long.MAX_VALUE;
        long firstTimestamp = -1;
        long lastTimestamp = -1;
        int numSamples = 0;
        itr = mmStore.setupIterator(query, subQuery, Collections.singletonList(Authorizations.EMPTY),
                Long.MAX_VALUE);
        while (itr.hasTop()) {
            itr.next();
            Map<Set<Tag>, Aggregation> aggregations = AggregationIterator.decodeValue(itr.getTopValue());
            for (Map.Entry<Set<Tag>, Aggregation> entry : aggregations.entrySet()) {
                for (Sample s : entry.getValue()) {
                    numSamples++;
                    if (firstTimestamp == -1) {
                        firstTimestamp = s.timestamp;
                    }
                    lastTimestamp = s.timestamp;
                    // if (s.timestamp < firstTimestamp) {
                    // firstTimestamp = s.timestamp;
                    // }
                    // if (s.timestamp > lastTimestamp) {
                    // lastTimestamp = s.timestamp;
                    // }
                }
            }
        }
        Assert.assertEquals("First timestamp incorrect", BASETIME + 1000, firstTimestamp);
        Assert.assertEquals("Last timestamp incorrect", BASETIME + 1440000, lastTimestamp);
        Assert.assertEquals("Number of samples incorrect", 2880, numSamples);
    } catch (IOException | ClassNotFoundException e) {
        LOG.error("exception in test", e);
    }
}
 

@Test
public void testDownsampleIterator() throws TimelyException {

    long BASETIME = System.currentTimeMillis();
    // align basetime to a downsample period
    BASETIME = BASETIME - (BASETIME % (1000 * 60));
    DataStoreCache mmStore = getMetricMemoryStore1(BASETIME);

    QueryRequest query = new QueryRequest();
    query.setStart(BASETIME);
    query.setEnd(BASETIME + 1440000);
    query.setMsResolution(true);
    QueryRequest.SubQuery subQuery = new QueryRequest.SubQuery();
    subQuery.setDownsample(Optional.of("1m-avg"));
    subQuery.setMetric("metric.number.1");
    subQuery.addTag("host", ".*");
    query.setQueries(Collections.singleton(subQuery));

    SortedKeyValueIterator<org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value> itr = null;
    try {
        long firstTimestamp = -1;
        long lastTimestamp = -1;
        int numSamples = 0;
        itr = mmStore.setupIterator(query, subQuery, Collections.singletonList(Authorizations.EMPTY),
                Long.MAX_VALUE);
        while (itr.hasTop()) {
            itr.next();
            Map<Set<Tag>, Aggregation> aggregations = AggregationIterator.decodeValue(itr.getTopValue());
            for (Map.Entry<Set<Tag>, Aggregation> entry : aggregations.entrySet()) {
                for (Sample s : entry.getValue()) {
                    numSamples++;
                    if (firstTimestamp == -1) {
                        firstTimestamp = s.timestamp;
                    }
                    lastTimestamp = s.timestamp;
                }
            }
        }
        Assert.assertEquals("First timestamp incorrect", BASETIME, firstTimestamp);
        Assert.assertEquals("Last timestamp incorrect", BASETIME + 1440000, lastTimestamp);
        Assert.assertEquals("Number of samples incorrect", 50, numSamples);
    } catch (IOException | ClassNotFoundException e) {
        LOG.error("exception in test", e);
    }
}
 

public void next() throws IOException {

        // always start fresh
        this.setTopKey(null);

        if (log.isDebugEnabled()) {
            TreeNode[] path = this.getPath();
            log.debug("BLTNODE.next() path-> " + this.buildTreePathString(path));
        }

        // have I been marked as done?
        if (this.isDone()) {
            if (log.isDebugEnabled()) {
                log.debug("I've been marked as done, returning");
            }
            return;
        }

        SortedKeyValueIterator<?, ?> iter = (SortedKeyValueIterator<?, ?>) this.getUserObject();
        iter.next();

        if (iter.hasTop()) {
            Key key = (Key) iter.getTopKey();

            // I have a valid topKey, pull out the piece I want
            key = buildKey(key);
            this.setTopKey(key);

            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.next() -> found: " + this.getTopKey());
            }
        } else {
            // no top value has been returned, I'm done.
            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.next() -> Nothing found");
            }
            this.setTopKey(null);
            this.setDone(true);
        }

    }
 

public static String stringTopKey(SortedKeyValueIterator<Key,Value> iter) {
  if (iter.hasTop())
    return iter.getTopKey().toString();
  return "";
}
 

public static String stringTopKey(final SortedKeyValueIterator<Key, Value> iter) {
	if (iter.hasTop()) {
           return iter.getTopKey().toString();
       }
	return "";
}
 

@Override
public Key apply(SortedKeyValueIterator<Key,Value> itr, Document doc, AttributeFactory attrs) throws IOException {
    Key key = itr.getTopKey();
    Text row = key.getRow();
    ByteSequence pointer = parsePointer(key.getColumnQualifierData());
    Key nextKey = key;
    while (nextKey != null && samePointer(row, pointer, nextKey)) {
        DatawaveKey topKey = new DatawaveKey(nextKey);
        String field = topKey.getFieldName();
        String value = topKey.getFieldValue();
        
        FieldValueCardinality fvC = null;
        byte[] currentValue = itr.getTopValue().get();
        try {
            if (currentValue.length > 0) {
                fvC = new FieldValueCardinality(HyperLogLogPlus.Builder.build(currentValue));
                if (log.isTraceEnabled()) {
                    log.trace("Set cardinality from FI value");
                }
            }
        } catch (Exception e) {
            if (log.isTraceEnabled()) {
                log.trace("Exception encountered " + e);
            }
        }
        
        if (null == fvC) {
            if (log.isTraceEnabled())
                log.trace("Building cardinality for " + topKey.getUid());
            fvC = new FieldValueCardinality();
            if (setDocIds)
                fvC.setDocId(topKey.getUid());
        }
        
        fvC.setContent(value);
        
        // for cardinalities, only use the visibility metadata
        Key metadata = new Key(EMPTY_TEXT, EMPTY_TEXT, EMPTY_TEXT, itr.getTopKey().getColumnVisibility(), -1);
        
        Cardinality card = new Cardinality(fvC, metadata, doc.isToKeep());
        
        // only keep fields that are index only
        card.setToKeep(fieldsToKeep == null || fieldsToKeep.contains(JexlASTHelper.removeGroupingContext(field)));
        doc.put(field, card);
        
        key = nextKey;
        itr.next();
        nextKey = (itr.hasTop() ? itr.getTopKey() : null);
    }
    return TLD.buildParentKey(row, pointer, TLD.parseFieldAndValueFromFI(key.getColumnFamilyData(), key.getColumnQualifierData()),
                    key.getColumnVisibility(), key.getTimestamp());
}
 

/**
 * Given a Key pointing to the start of an document to aggregate, construct a Range that should encapsulate the "document" to be aggregated together. Also
 * checks to see if data was found for the constructed Range before returning.
 * 
 * @param documentStartKey
 *            A Key of the form "bucket type\x00uid: "
 * @param keyRange
 *            the Range used to initialize source with seek()
 * @return the attributes
 */
private static List<Entry<Key,Value>> collectAttributesForDocumentKey(Key documentStartKey, SortedKeyValueIterator<Key,Value> source, Equality equality,
                EventDataQueryFilter filter, Set<Key> docKeys, Range keyRange) throws IOException {
    
    // setup the document key we are filtering for on the EventDataQueryFilter
    if (filter != null) {
        filter.startNewDocument(documentStartKey);
    }
    
    final List<Entry<Key,Value>> documentAttributes;
    if (null == documentStartKey) {
        documentAttributes = Collections.emptyList();
    } else {
        documentAttributes = new ArrayList<>(256);
        WeakReference<Key> docAttrKey = new WeakReference<>(source.getTopKey());
        
        while (docAttrKey != null) {
            boolean seeked = false;
            if (equality.partOf(documentStartKey, docAttrKey.get())) {
                if (filter == null || filter.keep(docAttrKey.get())) {
                    docKeys.add(getDocKey(docAttrKey.get()));
                }
                
                if (filter == null || filter.apply(Maps.immutableEntry(docAttrKey.get(), StringUtils.EMPTY))) {
                    documentAttributes.add(Maps.immutableEntry(docAttrKey.get(), source.getTopValue()));
                } else if (filter != null) {
                    Key limitKey = filter.transform(docAttrKey.get());
                    if (limitKey != null) {
                        documentAttributes.add(Maps.immutableEntry(limitKey, EMPTY_VALUE));
                    }
                    // request a seek range from the filter
                    Range seekRange = filter.getSeekRange(docAttrKey.get(), keyRange.getEndKey(), keyRange.isEndKeyInclusive());
                    if (seekRange != null) {
                        source.seek(seekRange, columnFamilies, inclusive);
                        seeked = true;
                    }
                }
            }
            
            // only call next if this wasn't a fresh seek()
            if (!seeked) {
                source.next();
            }
            
            if (source.hasTop()) {
                docAttrKey = new WeakReference<>(source.getTopKey());
            } else {
                docAttrKey = null;
            }
            
        }
    }
    
    return documentAttributes;
}