Java Code Examples for org.apache.accumulo.core.iterators.SortedKeyValueIterator#seek()

public TestData(SortedKeyValueIterator<Key,Value> iter, Range range, boolean reseek) {
    try {
        iter.seek(range, new HashSet<>(), false);
        
        while (iter.hasTop()) {
            data.put(iter.getTopKey(), iter.getTopValue());
            if (reseek) {
                iter.seek(new Range(iter.getTopKey(), false, range.getEndKey(), range.isEndKeyInclusive()), new HashSet<>(), false);
            } else {
                iter.next();
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void apply_testNormal() throws IOException {
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    Key result = aggregator.apply(itr);
    
    assertFalse(itr.hasTop());
    
    itr.seek(new Range(), null, true);
    Key result2 = aggregator.apply(itr, new Range(), null, false);
    
    assertFalse(itr.hasTop());
    assertEquals(result, result2);
}
 

@Test
public void apply_testNormal() throws IOException {
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getFi("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    Key result = aggregator.apply(itr);
    
    assertFalse(itr.hasTop());
    
    itr.seek(new Range(), null, true);
    Key result2 = aggregator.apply(itr, new Range(), null, false);
    
    assertFalse(itr.hasTop());
    assertEquals(result, result2);
}
 

/**
 * Advance an iterator until skip(...) returns false. May be a combination of seek() and next() calls
 * 
 * @param itr
 * @param pointer
 * @param currentRange
 * @param columnFamilies
 * @param includeColumnFamilies
 * @throws IOException
 */
protected void advanceItr(SortedKeyValueIterator<Key,Value> itr, ByteSequence pointer, Range currentRange, Collection<ByteSequence> columnFamilies,
                boolean includeColumnFamilies) throws IOException {
    Key current = itr.getTopKey();
    Text row = current.getRow();
    int nextCount = 0;
    while (current != null && skip(current, row, pointer)) {
        if (maxNextCount == -1 || nextCount < maxNextCount) {
            itr.next();
            nextCount++;
        } else {
            Key startKey = getSeekStartKey(current, pointer);
            Range newRange = new Range(startKey, false, currentRange.getEndKey(), currentRange.isEndKeyInclusive());
            itr.seek(newRange, columnFamilies, includeColumnFamilies);
            nextCount = 0;
        }
        
        current = itr.hasTop() ? itr.getTopKey() : null;
    }
}
 

/**
 * A bit of a hack, similar to the ColumnFamilySkippingIterator. This will call next a 32 times before finally seeking for the next {@code <row, colf>}.
 * 
 * The source iterator may or may not have a top after this method returns, and there is no guarantee of another viable top key/value being set.
 */
public static void moveTo(Key key, SortedKeyValueIterator<Key,Value> iterator, Range scanRange, Collection<ByteSequence> cfs, boolean inclusive)
                throws IOException {
    int nexts = 0;
    boolean movedEnough = false;
    if (log.isTraceEnabled()) {
        log.trace("Iterator key is " + key);
        log.trace("Iterator has top?" + iterator.hasTop());
        if (iterator.hasTop())
            log.trace("Iterator top key?" + iterator.getTopKey());
        log.trace("ScanRange is ?" + scanRange);
        log.trace("Scan cfs are " + cfs + " is inclusive ? " + inclusive);
    }
    while (nexts < 32 && iterator.hasTop() && (movedEnough = iterator.getTopKey().compareTo(key) < 0)) {
        iterator.next();
        nexts++;
    }
    if (log.isTraceEnabled()) {
        log.trace("Nexts is " + nexts);
        log.trace("Moved enough? " + movedEnough);
    }
    if (!movedEnough && iterator.hasTop()) {
        iterator.seek(new Range(key, true, scanRange.getEndKey(), scanRange.isEndKeyInclusive()), cfs, inclusive);
    }
}
 

@Test
public void apply_testNormal() throws IOException {
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    Key result = aggregator.apply(itr);
    
    assertFalse(itr.hasTop());
    
    itr.seek(new Range(), null, true);
    Key result2 = aggregator.apply(itr, new Range(), null, false);
    
    assertFalse(itr.hasTop());
    assertTrue(result.equals(result2));
}
 

/**
 * @param kvIter
 * @param node
 * @throws IOException
 */
@Override
protected void seekIndexOnlyDocument(SortedKeyValueIterator<Key,Value> kvIter, ASTEQNode node) throws IOException {
    if (null != rangeLimiter && limitLookup) {
        
        Key newStartKey = getKey(node);
        
        kvIter.seek(new Range(newStartKey, true, newStartKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false), Collections.emptyList(), false);
        
    }
}
 

@Test
public void dataIntegrity_individualTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // individual next loops
    int copy1Count = 0;
    while (copy1.hasTop()) {
        assertNotEquals("unexpected topKey on iteration=" + copy1Count, null, copy1.getTopKey());
        copy1Count++;
        copy1.next();
    }
    
    int copy2Count = 0;
    while (copy2.hasTop()) {
        assertNotEquals("unexpected topKey on iteration=" + copy2Count, null, copy2.getTopKey());
        copy2Count++;
        copy2.next();
    }
    
    assertTrue("both copies should have the same number of next calls; copy1=" + copy1Count + " copy2=" + copy2Count, copy1Count == copy2Count);
}
 

@Test
public void apply_buildDocKeep() throws IOException, ParseException {
    Document doc = new Document();
    AttributeFactory attributeFactory = new AttributeFactory(new TypeMetadata());
    
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    treeMap.put(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    
    Set<String> keepFields = new HashSet<>();
    keepFields.add("FIELD1");
    
    EventDataQueryFilter filter = new EventDataQueryFieldFilter(JexlASTHelper.parseJexlQuery("FIELD1 == 'VALUE1'"), Collections.emptySet());
    aggregator = new TermFrequencyAggregator(keepFields, filter, -1);
    Key result = aggregator.apply(itr, doc, attributeFactory);
    
    // test result key
    assertTrue(result != null);
    DatawaveKey parsedResult = new DatawaveKey(result);
    assertTrue(parsedResult.getDataType().equals("dataType1"));
    assertTrue(parsedResult.getUid().equals("123.345.456"));
    assertTrue(parsedResult.getFieldName().equals("FIELD1"));
    assertTrue(parsedResult.getFieldValue().equals("VALUE1"));
    
    // test that the doc is empty
    assertTrue(doc.size() == 2);
    assertTrue(doc.get("RECORD_ID").getData().equals("123/dataType1/123.345.456"));
    assertTrue(doc.get("FIELD1").getData().toString().equals("VALUE1"));
    
    // test that the iterator is in the correct position
    assertTrue(itr.hasTop());
    assertTrue(itr.getTopKey().equals(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10)));
}
 

public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {

        // always start fresh
        this.setTopKey(null);
        this.setDone(false);

        // get my user object which should be an iterator
        SortedKeyValueIterator<?, ?> iter = (SortedKeyValueIterator<?, ?>) this.getUserObject();
        if (iter != null) {

            iter.seek(range, columnFamilies, inclusive);

            if (iter.hasTop()) {
                Key key = (Key) iter.getTopKey();
                key = buildKey(key);

                this.setTopKey(key);
                if (log.isDebugEnabled()) {
                    log.debug("BLTNODE.seek() -> found: " + this.getTopKey());
                }
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("BLTNODE.seek() -> hasTop::false");
                }
                this.setDone(true);
            }
        } else {
            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.seek(), The iterator was null!");
            }
            this.setTopKey(null);
        }
    }
 

@Test
public void apply_buildDocOnlyKeepToKeep() throws IOException, ParseException {
    Document doc = new Document();
    AttributeFactory attributeFactory = new AttributeFactory(new TypeMetadata());
    
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456.1", 10), new Value());
    treeMap.put(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    
    Set<String> keepFields = new HashSet<>();
    keepFields.add("FIELD2");
    
    EventDataQueryFilter filter = new EventDataQueryFieldFilter(JexlASTHelper.parseJexlQuery("FIELD2 == 'VALUE1'"), Collections.emptySet());
    aggregator = new TLDTermFrequencyAggregator(keepFields, filter, -1);
    Key result = aggregator.apply(itr, doc, attributeFactory);
    
    // test result key
    assertTrue(result == null);
    
    // test that the doc is empty
    assertTrue(doc.size() == 0);
    
    // test that the iterator is in the correct position
    assertTrue(itr.hasTop());
    assertTrue(itr.getTopKey().equals(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10)));
}
 

@Test
public void apply_buildDocNotKeep() throws IOException, ParseException {
    Document doc = new Document();
    AttributeFactory attributeFactory = new AttributeFactory(new TypeMetadata());
    
    TreeMap<Key,Value> treeMap = Maps.newTreeMap();
    treeMap.put(getTF("123", "FIELD1", "VALUE1", "dataType1", "123.345.456", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.1", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.2", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.3", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.4", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.5", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.6", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.7", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.8", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.9", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.10.1", 10), new Value());
    treeMap.put(getTF("123", "FIELD1", "VALUE2", "dataType1", "123.345.456.11.1.1", 10), new Value());
    treeMap.put(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10), new Value());
    
    SortedKeyValueIterator<Key,Value> itr = new SortedMapIterator(treeMap);
    itr.seek(new Range(), null, true);
    
    Set<String> keepFields = new HashSet<>();
    keepFields.add("FIELD2");
    
    EventDataQueryFilter filter = new EventDataQueryFieldFilter(JexlASTHelper.parseJexlQuery("FIELD2 == 'abc'"), Collections.emptySet());
    aggregator = new TLDTermFrequencyAggregator(keepFields, filter, -1);
    Key result = aggregator.apply(itr, doc, attributeFactory);
    
    // test result key
    assertTrue(result == null);
    
    // test that the doc is empty
    assertTrue(doc.size() == 0);
    
    // test that the iterator is in the correct position
    assertTrue(itr.hasTop());
    assertTrue(itr.getTopKey().equals(getTF("123", "NEXT_DOC_FIELD", "VALUE1", "dataType1", "124.345.456", 10)));
}
 

@Test
public void dataIntegrity_emptyRangeTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // test one side empty range
    copy1.seek(new Range(new Key("20121126_9"), true, new Key("20121126_99"), false), Collections.emptyList(), false);
    copy2.seek(new Range(new Key("20121126_0"), true, new Key("20121126_4"), true), Collections.emptyList(), false);
    
    int mixedCopy1Count = 0;
    int mixedCopy2Count = 0;
    
    while (copy1.hasTop() || copy2.hasTop()) {
        if (copy1.hasTop()) {
            copy1.getTopKey();
            mixedCopy1Count++;
            copy1.next();
        }
        
        if (copy2.hasTop()) {
            copy2.getTopKey();
            mixedCopy2Count++;
            copy2.next();
        }
    }
    
    assertTrue(mixedCopy2Count > mixedCopy1Count);
    assertTrue(mixedCopy2Count == 26);
    assertTrue(mixedCopy1Count == 0);
}
 

protected void seek(SortedKeyValueIterator<Key,Value> source, Range r) throws IOException {
    if (log.isTraceEnabled()) {
        log.trace(this + " seek'ing to: " + r);
    }
    source.seek(r, this.seekColumnFamilies, true);
    next();
}
 

private Map<Set<Tag>, Aggregation> runQuery(SortedKeyValueIterator<Key, Value> iter, SortedMap<Key, Value> testData,
        long period) throws Exception {
    IteratorSetting is = new IteratorSetting(100, AggregationIterator.class);
    AggregationIterator.setAggregationOptions(is, Collections.singletonMap("host", ".*"), Avg.class.getName());
    SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(testData);
    iter.init(source, is.getOptions(), null);
    iter.seek(new Range(), Collections.emptyList(), true);
    assertTrue(iter.hasTop());
    Key key = iter.getTopKey();
    assertEquals(testData.lastKey(), key);
    Map<Set<Tag>, Aggregation> samples = AggregationIterator.decodeValue(iter.getTopValue());
    return samples;
}
 

protected SortedKeyValueIterator<Key,Value> createIndexOnlyKey(ASTEQNode node) throws IOException {
    Key newStartKey = getKey(node);
    
    IdentifierOpLiteral op = JexlASTHelper.getIdentifierOpLiteral(node);
    if (null == op || null == op.getLiteralValue()) {
        // deep copy since this is likely a null literal
        return source.deepCopy(env);
    }
    
    String fn = op.deconstructIdentifier();
    String literal = String.valueOf(op.getLiteralValue());
    
    if (log.isTraceEnabled()) {
        log.trace("createIndexOnlyKey for " + fn + " " + literal + " " + newStartKey);
    }
    List<Entry<Key,Value>> kv = Lists.newArrayList();
    if (null != limitedMap.get(Maps.immutableEntry(fn, literal))) {
        kv.add(limitedMap.get(Maps.immutableEntry(fn, literal)));
    } else {
        
        SortedKeyValueIterator<Key,Value> mySource = limitedSource;
        // if source size > 0, we are free to use up to that number for this query
        if (source.getSourceSize() > 0)
            mySource = source.deepCopy(env);
        
        mySource.seek(new Range(newStartKey, true, newStartKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false), Collections.emptyList(), false);
        
        if (mySource.hasTop()) {
            kv.add(Maps.immutableEntry(mySource.getTopKey(), Constants.NULL_VALUE));
            
        }
    }
    
    return new IteratorToSortedKeyValueIterator(kv.iterator());
}
 

@Override
protected void seekIndexOnlyDocument(SortedKeyValueIterator<Key,Value> kvIter, ASTEQNode node) throws IOException {
    if (null != rangeLimiter && limitLookup) {
        Key startKey = getKey(node);
        Key endKey = getEndKey(node);
        
        kvIter.seek(new Range(startKey, true, endKey, true), Collections.emptyList(), false);
    }
}
 

@Test
public void dataIntegrity_alternatingTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // re-seek
    copy1.seek(new Range(), Collections.emptyList(), false);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // alternating next loops
    int alternatingCount = 0;
    while (copy1.hasTop() && copy2.hasTop()) {
        assertTrue(copy1.getTopKey().equals(copy2.getTopKey()));
        alternatingCount++;
        copy1.next();
        copy2.next();
    }
    
    assertFalse(copy1.hasTop());
    assertFalse(copy2.hasTop());
    assertEquals(26, alternatingCount);
}
 

protected void seek(SortedKeyValueIterator<Key,Value> source, Range r) throws IOException {
    source.seek(r, this.seekColumnFamilies, true);
}
 

@Test
public void dataIntegrity_differentRangeReseekTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // different ranges
    copy1.seek(new Range(new Key("20121126_2"), true, new Key("20121126_3"), false), Collections.emptyList(), false);
    copy2.seek(new Range(new Key("20121126_0"), true, new Key("20121126_4"), true), Collections.emptyList(), false);
    
    int mixedCopy1Count = 0;
    int mixedCopy2Count = 0;
    boolean reseek = true;
    
    while (copy1.hasTop() || copy2.hasTop()) {
        if (copy1.hasTop()) {
            copy1.getTopKey();
            mixedCopy1Count++;
            copy1.next();
        }
        
        if (!copy1.hasTop() && reseek) {
            // test intermediate seek
            copy1.seek(new Range(new Key("20121126_2"), true, new Key("20121126_3"), false), Collections.emptyList(), false);
            reseek = false;
        }
        
        if (copy2.hasTop()) {
            copy2.getTopKey();
            mixedCopy2Count++;
            copy2.next();
        }
    }
    
    assertTrue(mixedCopy2Count > mixedCopy1Count);
    assertTrue(mixedCopy2Count == 26);
    // since re-seek after the first one should be 2x expected
    assertTrue(mixedCopy1Count == 9 * 2);
}