Java Code Examples for org.apache.accumulo.core.iterators.SortedKeyValueIterator#next()

The following examples show how to use org.apache.accumulo.core.iterators.SortedKeyValueIterator#next() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PropogatingIteratorSeekTest.java    From datawave with Apache License 2.0 6 votes vote down vote up
public TestData(SortedKeyValueIterator<Key,Value> iter, Range range, boolean reseek) {
    try {
        iter.seek(range, new HashSet<>(), false);
        
        while (iter.hasTop()) {
            data.put(iter.getTopKey(), iter.getTopValue());
            if (reseek) {
                iter.seek(new Range(iter.getTopKey(), false, range.getEndKey(), range.isEndKeyInclusive()), new HashSet<>(), false);
            } else {
                iter.next();
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 2
Source File: QueryFilterIterator.java    From geowave with Apache License 2.0 6 votes vote down vote up
protected static void findTopEnhanced(
    final SortedKeyValueIterator<Key, Value> source,
    final Filter filter) {
  Key key;
  if (source.hasTop()) {
    key = source.getTopKey();
  } else {
    return;
  }
  while (!key.isDeleted() && !filter.accept(key, source.getTopValue())) {
    try {
      source.next();
      if (source.hasTop()) {
        key = source.getTopKey();
      } else {
        return;
      }
    } catch (final IOException e) {
      throw new RuntimeException(e);
    }
  }
}
 
Example 3
Source File: SeekingAggregator.java    From datawave with Apache License 2.0 6 votes vote down vote up
/**
 * Advance an iterator until skip(...) returns false. May be a combination of seek() and next() calls
 * 
 * @param itr
 * @param pointer
 * @param currentRange
 * @param columnFamilies
 * @param includeColumnFamilies
 * @throws IOException
 */
protected void advanceItr(SortedKeyValueIterator<Key,Value> itr, ByteSequence pointer, Range currentRange, Collection<ByteSequence> columnFamilies,
                boolean includeColumnFamilies) throws IOException {
    Key current = itr.getTopKey();
    Text row = current.getRow();
    int nextCount = 0;
    while (current != null && skip(current, row, pointer)) {
        if (maxNextCount == -1 || nextCount < maxNextCount) {
            itr.next();
            nextCount++;
        } else {
            Key startKey = getSeekStartKey(current, pointer);
            Range newRange = new Range(startKey, false, currentRange.getEndKey(), currentRange.isEndKeyInclusive());
            itr.seek(newRange, columnFamilies, includeColumnFamilies);
            nextCount = 0;
        }
        
        current = itr.hasTop() ? itr.getTopKey() : null;
    }
}
 
Example 4
Source File: ChunkCombinerTest.java    From accumulo-examples with Apache License 2.0 6 votes vote down vote up
private void runTest(boolean reseek, TreeMap<Key,Value> source, TreeMap<Key,Value> result,
    Collection<ByteSequence> cols) throws IOException {
  MapIterator src = new MapIterator(source);
  SortedKeyValueIterator<Key,Value> iter = new ChunkCombiner();
  iter.init(src, null, null);
  iter = iter.deepCopy(null);
  iter.seek(new Range(), cols, true);

  TreeMap<Key,Value> seen = new TreeMap<>();

  while (iter.hasTop()) {
    assertFalse("already contains " + iter.getTopKey(), seen.containsKey(iter.getTopKey()));
    seen.put(new Key(iter.getTopKey()), new Value(iter.getTopValue()));

    if (reseek)
      iter.seek(new Range(iter.getTopKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL), true,
          null, true), cols, true);
    else
      iter.next();
  }

  assertEquals(result, seen);
}
 
Example 5
Source File: TestData.java    From fluo with Apache License 2.0 6 votes vote down vote up
TestData(SortedKeyValueIterator<Key, Value> iter, Range range, boolean reseek) {
  try {
    iter.seek(range, new HashSet<ByteSequence>(), false);

    while (iter.hasTop()) {
      data.put(iter.getTopKey(), iter.getTopValue());
      if (reseek) {
        iter.seek(
            new Range(iter.getTopKey(), false, range.getEndKey(), range.isEndKeyInclusive()),
            new HashSet<ByteSequence>(), false);
      } else {
        iter.next();
      }
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 6
Source File: UniqueColumnFamilyIterator.java    From datawave with Apache License 2.0 6 votes vote down vote up
/**
 * A bit of a hack, similar to the ColumnFamilySkippingIterator. This will call next a 32 times before finally seeking for the next {@code <row, colf>}.
 * 
 * The source iterator may or may not have a top after this method returns, and there is no guarantee of another viable top key/value being set.
 */
public static void moveTo(Key key, SortedKeyValueIterator<Key,Value> iterator, Range scanRange, Collection<ByteSequence> cfs, boolean inclusive)
                throws IOException {
    int nexts = 0;
    boolean movedEnough = false;
    if (log.isTraceEnabled()) {
        log.trace("Iterator key is " + key);
        log.trace("Iterator has top?" + iterator.hasTop());
        if (iterator.hasTop())
            log.trace("Iterator top key?" + iterator.getTopKey());
        log.trace("ScanRange is ?" + scanRange);
        log.trace("Scan cfs are " + cfs + " is inclusive ? " + inclusive);
    }
    while (nexts < 32 && iterator.hasTop() && (movedEnough = iterator.getTopKey().compareTo(key) < 0)) {
        iterator.next();
        nexts++;
    }
    if (log.isTraceEnabled()) {
        log.trace("Nexts is " + nexts);
        log.trace("Moved enough? " + movedEnough);
    }
    if (!movedEnough && iterator.hasTop()) {
        iterator.seek(new Range(key, true, scanRange.getEndKey(), scanRange.isEndKeyInclusive()), cfs, inclusive);
    }
}
 
Example 7
Source File: SourceManagerTest.java    From datawave with Apache License 2.0 6 votes vote down vote up
@Test
public void assertNexts() throws IOException {
    SourceManager manager = new SourceManager(counter);
    manager.setInitialSize(10);
    
    Collection<SortedKeyValueIterator<Key,Value>> kvList = Lists.newArrayList();
    for (int i = 0; i < 500; i++) {
        kvList.add(manager.deepCopy(null));
    }
    
    for (SortedKeyValueIterator<Key,Value> kv : kvList) {
        kv.seek(new Range(), Collections.emptyList(), false);
        kv.next();
    }
    
    assertEquals(500, counter.nextCalls);
    assertEquals(10, counter.counter);
}
 
Example 8
Source File: TLDFieldIndexAggregator.java    From datawave with Apache License 2.0 5 votes vote down vote up
public Key apply(SortedKeyValueIterator<Key,Value> itr, Document d, AttributeFactory af) throws IOException {
    Key key = itr.getTopKey();
    ByteSequence parentId = parseRootPointerFromFI(key.getColumnQualifierData());
    Text row = key.getRow();
    ByteSequence docId = null;
    Key nextKey = key;
    do {
        key = nextKey;
        String field = key.getColumnFamily().toString().substring(3);
        String value = key.getColumnQualifier().toString();
        value = value.substring(0, value.indexOf('\0'));
        Attribute<?> attr = af.create(field, value, key, true);
        // in addition to keeping fields that the filter indicates should be kept, also keep fields that the filter applies. This is due to inconsistent
        // behavior between event/tld queries where an index only field index will be kept except when it is a child of a tld
        attr.setToKeep((fieldsToAggregate == null || fieldsToAggregate.contains(JexlASTHelper.removeGroupingContext(field)))
                        && (attrFilter == null || attrFilter.keep(key)));
        d.put(field, attr);
        
        ByteSequence thisId = parsePointerFromFI(key.getColumnQualifierData());
        if (docId == null || !docId.equals(thisId)) {
            docId = thisId;
            Key docKey = new Key(key.getRow(), new Text(docId.toArray()), new Text(), ColumnVisibilityCache.get(key.getColumnVisibilityData()),
                            key.getTimestamp());
            attr = new DocumentKey(docKey, false);
            d.put(Document.DOCKEY_FIELD_NAME, attr);
        }
        itr.next();
        nextKey = itr.hasTop() ? itr.getTopKey() : null;
    } while (skip(nextKey, row, parentId));
    return getResult(key, parentId);
}
 
Example 9
Source File: SecondaryIndexQueryFilterIterator.java    From geowave with Apache License 2.0 5 votes vote down vote up
@Override
public boolean acceptRow(final SortedKeyValueIterator<Key, Value> rowIterator)
    throws IOException {
  if (filter != null) {
    while (rowIterator.hasTop()) {
      final Key key = rowIterator.getTopKey();
      final Value value = rowIterator.getTopValue();
      final String cq =
          StringUtils.stringFromBinary(key.getColumnQualifierData().getBackingArray());
      if (!cq.equals(primaryIndexId)) {
        final IndexedPersistenceEncoding<ByteArray> persistenceEncoding =
            new IndexedPersistenceEncoding<>(
                null, // not needed
                null, // not needed
                null, // not needed
                null, // not needed
                0, // not needed
                new MultiFieldPersistentDataset<>(
                    StringUtils.stringFromBinary(key.getColumnQualifierData().getBackingArray()),
                    new ByteArray(value.get())),
                null);
        if (filter.accept(null, persistenceEncoding)) {
          return true;
        }
      }
      rowIterator.next();
    }
    return false;
  }
  // should not happen but if the filter is not sent to this iterator, it
  // will accept everything
  return true;
}
 
Example 10
Source File: SourceManagerTest.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Test
public void dataIntegrity_emptyRangeTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // test one side empty range
    copy1.seek(new Range(new Key("20121126_9"), true, new Key("20121126_99"), false), Collections.emptyList(), false);
    copy2.seek(new Range(new Key("20121126_0"), true, new Key("20121126_4"), true), Collections.emptyList(), false);
    
    int mixedCopy1Count = 0;
    int mixedCopy2Count = 0;
    
    while (copy1.hasTop() || copy2.hasTop()) {
        if (copy1.hasTop()) {
            copy1.getTopKey();
            mixedCopy1Count++;
            copy1.next();
        }
        
        if (copy2.hasTop()) {
            copy2.getTopKey();
            mixedCopy2Count++;
            copy2.next();
        }
    }
    
    assertTrue(mixedCopy2Count > mixedCopy1Count);
    assertTrue(mixedCopy2Count == 26);
    assertTrue(mixedCopy1Count == 0);
}
 
Example 11
Source File: SourceManagerTest.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Test
public void dataIntegrity_individualTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // individual next loops
    int copy1Count = 0;
    while (copy1.hasTop()) {
        assertNotEquals("unexpected topKey on iteration=" + copy1Count, null, copy1.getTopKey());
        copy1Count++;
        copy1.next();
    }
    
    int copy2Count = 0;
    while (copy2.hasTop()) {
        assertNotEquals("unexpected topKey on iteration=" + copy2Count, null, copy2.getTopKey());
        copy2Count++;
        copy2.next();
    }
    
    assertTrue("both copies should have the same number of next calls; copy1=" + copy1Count + " copy2=" + copy2Count, copy1Count == copy2Count);
}
 
Example 12
Source File: IdentityAggregator.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Override
public Key apply(SortedKeyValueIterator<Key,Value> itr) throws IOException {
    Key key = itr.getTopKey();
    Text row = key.getRow();
    ByteSequence pointer = parsePointer(key.getColumnQualifierData());
    while (itr.hasTop() && samePointer(row, pointer, itr.getTopKey()))
        itr.next();
    
    return TLD.buildParentKey(row, pointer, parseFieldNameValue(key.getColumnFamilyData(), key.getColumnQualifierData()), key.getColumnVisibility(),
                    key.getTimestamp());
}
 
Example 13
Source File: TLDFieldIndexAggregator.java    From datawave with Apache License 2.0 5 votes vote down vote up
public Key apply(SortedKeyValueIterator<Key,Value> itr) throws IOException {
    Key key = itr.getTopKey();
    ByteSequence parentId = parsePointer(key);
    Text row = key.getRow();
    Key nextKey = key;
    do {
        key = nextKey;
        itr.next();
        nextKey = (itr.hasTop() ? itr.getTopKey() : null);
    } while (skip(nextKey, row, parentId));
    return getResult(key, parentId);
}
 
Example 14
Source File: SourceManagerTest.java    From datawave with Apache License 2.0 4 votes vote down vote up
@Test
public void dataIntegrity_differentRangeReseekTest() throws IOException {
    SourceManager manager = new SourceManager(dataIterator);
    manager.setInitialSize(1);
    
    // pre-seek both iterators
    SortedKeyValueIterator<Key,Value> copy1 = manager.deepCopy(null);
    copy1.seek(new Range(), Collections.emptyList(), false);
    SortedKeyValueIterator<Key,Value> copy2 = manager.deepCopy(null);
    copy2.seek(new Range(), Collections.emptyList(), false);
    
    // different ranges
    copy1.seek(new Range(new Key("20121126_2"), true, new Key("20121126_3"), false), Collections.emptyList(), false);
    copy2.seek(new Range(new Key("20121126_0"), true, new Key("20121126_4"), true), Collections.emptyList(), false);
    
    int mixedCopy1Count = 0;
    int mixedCopy2Count = 0;
    boolean reseek = true;
    
    while (copy1.hasTop() || copy2.hasTop()) {
        if (copy1.hasTop()) {
            copy1.getTopKey();
            mixedCopy1Count++;
            copy1.next();
        }
        
        if (!copy1.hasTop() && reseek) {
            // test intermediate seek
            copy1.seek(new Range(new Key("20121126_2"), true, new Key("20121126_3"), false), Collections.emptyList(), false);
            reseek = false;
        }
        
        if (copy2.hasTop()) {
            copy2.getTopKey();
            mixedCopy2Count++;
            copy2.next();
        }
    }
    
    assertTrue(mixedCopy2Count > mixedCopy1Count);
    assertTrue(mixedCopy2Count == 26);
    // since re-seek after the first one should be 2x expected
    assertTrue(mixedCopy1Count == 9 * 2);
}
 
Example 15
Source File: BooleanLogicTreeNode.java    From accumulo-recipes with Apache License 2.0 4 votes vote down vote up
public void next() throws IOException {

        // always start fresh
        this.setTopKey(null);

        if (log.isDebugEnabled()) {
            TreeNode[] path = this.getPath();
            log.debug("BLTNODE.next() path-> " + this.buildTreePathString(path));
        }

        // have I been marked as done?
        if (this.isDone()) {
            if (log.isDebugEnabled()) {
                log.debug("I've been marked as done, returning");
            }
            return;
        }

        SortedKeyValueIterator<?, ?> iter = (SortedKeyValueIterator<?, ?>) this.getUserObject();
        iter.next();

        if (iter.hasTop()) {
            Key key = (Key) iter.getTopKey();

            // I have a valid topKey, pull out the piece I want
            key = buildKey(key);
            this.setTopKey(key);

            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.next() -> found: " + this.getTopKey());
            }
        } else {
            // no top value has been returned, I'm done.
            if (log.isDebugEnabled()) {
                log.debug("BLTNODE.next() -> Nothing found");
            }
            this.setTopKey(null);
            this.setDone(true);
        }

    }
 
Example 16
Source File: TestDataStoreCacheIterator.java    From timely with Apache License 2.0 4 votes vote down vote up
@Test
public void testDownsampleIterator() throws TimelyException {

    long BASETIME = System.currentTimeMillis();
    // align basetime to a downsample period
    BASETIME = BASETIME - (BASETIME % (1000 * 60));
    DataStoreCache mmStore = getMetricMemoryStore1(BASETIME);

    QueryRequest query = new QueryRequest();
    query.setStart(BASETIME);
    query.setEnd(BASETIME + 1440000);
    query.setMsResolution(true);
    QueryRequest.SubQuery subQuery = new QueryRequest.SubQuery();
    subQuery.setDownsample(Optional.of("1m-avg"));
    subQuery.setMetric("metric.number.1");
    subQuery.addTag("host", ".*");
    query.setQueries(Collections.singleton(subQuery));

    SortedKeyValueIterator<org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value> itr = null;
    try {
        long firstTimestamp = -1;
        long lastTimestamp = -1;
        int numSamples = 0;
        itr = mmStore.setupIterator(query, subQuery, Collections.singletonList(Authorizations.EMPTY),
                Long.MAX_VALUE);
        while (itr.hasTop()) {
            itr.next();
            Map<Set<Tag>, Aggregation> aggregations = AggregationIterator.decodeValue(itr.getTopValue());
            for (Map.Entry<Set<Tag>, Aggregation> entry : aggregations.entrySet()) {
                for (Sample s : entry.getValue()) {
                    numSamples++;
                    if (firstTimestamp == -1) {
                        firstTimestamp = s.timestamp;
                    }
                    lastTimestamp = s.timestamp;
                }
            }
        }
        Assert.assertEquals("First timestamp incorrect", BASETIME, firstTimestamp);
        Assert.assertEquals("Last timestamp incorrect", BASETIME + 1440000, lastTimestamp);
        Assert.assertEquals("Number of samples incorrect", 50, numSamples);
    } catch (IOException | ClassNotFoundException e) {
        LOG.error("exception in test", e);
    }
}
 
Example 17
Source File: TestDataStoreCacheIterator.java    From timely with Apache License 2.0 4 votes vote down vote up
@Test
public void testRateIterator() throws TimelyException {

    long BASETIME = System.currentTimeMillis();
    // align basetime to a downsample period
    BASETIME = BASETIME - (BASETIME % 1000);
    DataStoreCache mmStore = getMetricMemoryStore2(BASETIME);

    QueryRequest query = new QueryRequest();
    query.setStart(BASETIME);
    query.setEnd(BASETIME + 1440000);
    query.setMsResolution(true);
    QueryRequest.SubQuery subQuery = new QueryRequest.SubQuery();
    subQuery.setDownsample(Optional.of("1ms-avg"));
    subQuery.setMetric("metric.number.1");
    subQuery.addTag("host", ".*");
    QueryRequest.RateOption rateOption = new QueryRequest.RateOption();
    rateOption.setCounter(false);
    subQuery.setRate(true);
    subQuery.setRateOptions(rateOption);
    query.setQueries(Collections.singleton(subQuery));

    SortedKeyValueIterator<org.apache.accumulo.core.data.Key, org.apache.accumulo.core.data.Value> itr = null;
    try {
        // long firstTimestamp = Long.MAX_VALUE;
        long firstTimestamp = -1;
        long lastTimestamp = -1;
        int numSamples = 0;
        itr = mmStore.setupIterator(query, subQuery, Collections.singletonList(Authorizations.EMPTY),
                Long.MAX_VALUE);
        while (itr.hasTop()) {
            itr.next();
            Map<Set<Tag>, Aggregation> aggregations = AggregationIterator.decodeValue(itr.getTopValue());
            for (Map.Entry<Set<Tag>, Aggregation> entry : aggregations.entrySet()) {
                for (Sample s : entry.getValue()) {
                    numSamples++;
                    if (firstTimestamp == -1) {
                        firstTimestamp = s.timestamp;
                    }
                    lastTimestamp = s.timestamp;
                    // if (s.timestamp < firstTimestamp) {
                    // firstTimestamp = s.timestamp;
                    // }
                    // if (s.timestamp > lastTimestamp) {
                    // lastTimestamp = s.timestamp;
                    // }
                }
            }
        }
        Assert.assertEquals("First timestamp incorrect", BASETIME + 1000, firstTimestamp);
        Assert.assertEquals("Last timestamp incorrect", BASETIME + 1440000, lastTimestamp);
        Assert.assertEquals("Number of samples incorrect", 2880, numSamples);
    } catch (IOException | ClassNotFoundException e) {
        LOG.error("exception in test", e);
    }
}
 
Example 18
Source File: CardinalityAggregator.java    From datawave with Apache License 2.0 4 votes vote down vote up
@Override
public Key apply(SortedKeyValueIterator<Key,Value> itr, Document doc, AttributeFactory attrs) throws IOException {
    Key key = itr.getTopKey();
    Text row = key.getRow();
    ByteSequence pointer = parsePointer(key.getColumnQualifierData());
    Key nextKey = key;
    while (nextKey != null && samePointer(row, pointer, nextKey)) {
        DatawaveKey topKey = new DatawaveKey(nextKey);
        String field = topKey.getFieldName();
        String value = topKey.getFieldValue();
        
        FieldValueCardinality fvC = null;
        byte[] currentValue = itr.getTopValue().get();
        try {
            if (currentValue.length > 0) {
                fvC = new FieldValueCardinality(HyperLogLogPlus.Builder.build(currentValue));
                if (log.isTraceEnabled()) {
                    log.trace("Set cardinality from FI value");
                }
            }
        } catch (Exception e) {
            if (log.isTraceEnabled()) {
                log.trace("Exception encountered " + e);
            }
        }
        
        if (null == fvC) {
            if (log.isTraceEnabled())
                log.trace("Building cardinality for " + topKey.getUid());
            fvC = new FieldValueCardinality();
            if (setDocIds)
                fvC.setDocId(topKey.getUid());
        }
        
        fvC.setContent(value);
        
        // for cardinalities, only use the visibility metadata
        Key metadata = new Key(EMPTY_TEXT, EMPTY_TEXT, EMPTY_TEXT, itr.getTopKey().getColumnVisibility(), -1);
        
        Cardinality card = new Cardinality(fvC, metadata, doc.isToKeep());
        
        // only keep fields that are index only
        card.setToKeep(fieldsToKeep == null || fieldsToKeep.contains(JexlASTHelper.removeGroupingContext(field)));
        doc.put(field, card);
        
        key = nextKey;
        itr.next();
        nextKey = (itr.hasTop() ? itr.getTopKey() : null);
    }
    return TLD.buildParentKey(row, pointer, TLD.parseFieldAndValueFromFI(key.getColumnFamilyData(), key.getColumnQualifierData()),
                    key.getColumnVisibility(), key.getTimestamp());
}
 
Example 19
Source File: MergingVisibilityCombiner.java    From geowave with Apache License 2.0 4 votes vote down vote up
@Override
protected void transformRangeInternal(
    final SortedKeyValueIterator<Key, Value> input,
    final KVBuffer output) throws IOException {
  Mergeable currentMergeable = null;
  Key outputKey = null;
  workKey.set(input.getTopKey());
  // default to not combining, only combine when combiners does not
  // contain this column
  if ((combiners == null) || !combiners.contains(workKey) || workKey.isDeleted()) {
    // don't transform at all
    while (input.hasTop()) {
      output.append(input.getTopKey(), input.getTopValue());
      input.next();
    }
    return;
  }
  while (input.hasTop()) {
    final Value val = input.getTopValue();
    // the SortedKeyValueIterator uses the same instance of topKey to
    // hold keys (a wrapper)
    final Key currentKey = new Key(input.getTopKey());
    if (outputKey == null) {
      outputKey = currentKey;
    } else if ((currentMergeable != null)
        && !outputKey.getRowData().equals(currentKey.getRowData())) {
      output.append(outputKey, new Value(URLClassloaderUtils.toBinary(currentMergeable)));
      currentMergeable = null;
      outputKey = currentKey;
      continue;
    } else {
      final Text combinedVisibility =
          new Text(
              combineVisibilities(
                  currentKey.getColumnVisibility().getBytes(),
                  outputKey.getColumnVisibility().getBytes()));
      outputKey = replaceColumnVisibility(outputKey, combinedVisibility);
    }
    final Mergeable mergeable = getMergeable(currentKey, val.get());
    // hopefully its never the case that null mergeables are stored,
    // but just in case, check
    if (mergeable != null) {
      if (currentMergeable == null) {
        currentMergeable = mergeable;
      } else {
        currentMergeable.merge(mergeable);
      }
    }
    input.next();
  }
  if (currentMergeable != null) {
    output.append(outputKey, new Value(getBinary(currentMergeable)));
  }
}
 
Example 20
Source File: KeyToDocumentData.java    From datawave with Apache License 2.0 4 votes vote down vote up
/**
 * Given a Key pointing to the start of an document to aggregate, construct a Range that should encapsulate the "document" to be aggregated together. Also
 * checks to see if data was found for the constructed Range before returning.
 * 
 * @param documentStartKey
 *            A Key of the form "bucket type\x00uid: "
 * @param keyRange
 *            the Range used to initialize source with seek()
 * @return the attributes
 */
private static List<Entry<Key,Value>> collectAttributesForDocumentKey(Key documentStartKey, SortedKeyValueIterator<Key,Value> source, Equality equality,
                EventDataQueryFilter filter, Set<Key> docKeys, Range keyRange) throws IOException {
    
    // setup the document key we are filtering for on the EventDataQueryFilter
    if (filter != null) {
        filter.startNewDocument(documentStartKey);
    }
    
    final List<Entry<Key,Value>> documentAttributes;
    if (null == documentStartKey) {
        documentAttributes = Collections.emptyList();
    } else {
        documentAttributes = new ArrayList<>(256);
        WeakReference<Key> docAttrKey = new WeakReference<>(source.getTopKey());
        
        while (docAttrKey != null) {
            boolean seeked = false;
            if (equality.partOf(documentStartKey, docAttrKey.get())) {
                if (filter == null || filter.keep(docAttrKey.get())) {
                    docKeys.add(getDocKey(docAttrKey.get()));
                }
                
                if (filter == null || filter.apply(Maps.immutableEntry(docAttrKey.get(), StringUtils.EMPTY))) {
                    documentAttributes.add(Maps.immutableEntry(docAttrKey.get(), source.getTopValue()));
                } else if (filter != null) {
                    Key limitKey = filter.transform(docAttrKey.get());
                    if (limitKey != null) {
                        documentAttributes.add(Maps.immutableEntry(limitKey, EMPTY_VALUE));
                    }
                    // request a seek range from the filter
                    Range seekRange = filter.getSeekRange(docAttrKey.get(), keyRange.getEndKey(), keyRange.isEndKeyInclusive());
                    if (seekRange != null) {
                        source.seek(seekRange, columnFamilies, inclusive);
                        seeked = true;
                    }
                }
            }
            
            // only call next if this wasn't a fresh seek()
            if (!seeked) {
                source.next();
            }
            
            if (source.hasTop()) {
                docAttrKey = new WeakReference<>(source.getTopKey());
            } else {
                docAttrKey = null;
            }
            
        }
    }
    
    return documentAttributes;
}