org.apache.lucene.util.Counter Java Examples

private void readFromBytes(BytesRef bytes) {
  // Read pruned flag
  this.setIsPruned(bytes.bytes[bytes.offset++] == 1 ? true : false);

  // Read size fo the set
  int size = Bytes.readInt(bytes);

  // Read terms
  bytesUsed = Counter.newCounter();
  pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  set = new BytesRefHash(pool);

  BytesRef reusable = new BytesRef();
  for (int i = 0; i < size; i++) {
    Bytes.readBytesRef(bytes, reusable);
    set.add(reusable);
  }
}
 

public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
                            Engine.Searcher engineSearcher, IndexService indexService, IndexShard indexShard,
                            ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
                            BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher,
                            TimeValue timeout
) {
    super(parseFieldMatcher, request);
    this.id = id;
    this.request = request;
    this.searchType = request.searchType();
    this.shardTarget = shardTarget;
    this.engineSearcher = engineSearcher;
    this.scriptService = scriptService;
    this.pageCacheRecycler = pageCacheRecycler;
    // SearchContexts use a BigArrays that can circuit break
    this.bigArrays = bigArrays.withCircuitBreaking();
    this.dfsResult = new DfsSearchResult(id, shardTarget);
    this.queryResult = new QuerySearchResult(id, shardTarget);
    this.fetchResult = new FetchSearchResult(id, shardTarget);
    this.indexShard = indexShard;
    this.indexService = indexService;
    this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
    this.timeEstimateCounter = timeEstimateCounter;
    this.timeoutInMillis = timeout.millis();
}
 

public void testSortAndDedupNumericUpdatesByTerms() throws IOException {
  List<DocValuesUpdate.NumericDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  String termField = RandomPicks.randomFrom(random(), Arrays.asList("id", "_id", "some_other_field"));
  long docValue = 1 + random().nextInt(1000);
  DocValuesUpdate.NumericDocValuesUpdate randomUpdate = new DocValuesUpdate.NumericDocValuesUpdate(
      new Term(termField, Integer.toString(random().nextInt(1000))), "numeric", docValue);
  randomUpdate = randomUpdate.prepareForApply(randomDocUpTo());
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = new DocValuesUpdate.NumericDocValuesUpdate(
        new Term(termField, Integer.toString(random().nextInt(1000))), "numeric", docValue);
    randomUpdate = randomUpdate.prepareForApply(randomDocUpTo());
    updates.add(randomUpdate);
    buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
  }
  buffer.finish();
  assertBufferUpdates(buffer, updates, true);
}
 

public CrateSearchContext(long id,
                          final long nowInMillis,
                          SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher,
                          IndexService indexService,
                          final IndexShard indexShard,
                          ScriptService scriptService,
                          PageCacheRecycler pageCacheRecycler,
                          BigArrays bigArrays,
                          Counter timeEstimateCounter,
                          Optional<Scroll> scroll) {
    super(id, new CrateSearchShardRequest(nowInMillis, scroll, indexShard),
            shardTarget, engineSearcher, indexService,
            indexShard, scriptService, pageCacheRecycler,
            bigArrays, timeEstimateCounter, ParseFieldMatcher.STRICT, SearchService.NO_TIMEOUT);
    this.engineSearcher = engineSearcher;
}
 

public void testNumericRandom() throws IOException {
  List<DocValuesUpdate.NumericDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  DocValuesUpdate.NumericDocValuesUpdate randomUpdate = getRandomNumericUpdate();
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = getRandomNumericUpdate();
    updates.add(randomUpdate);
    if (randomUpdate.hasValue) {
      buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
    } else {
      buffer.addNoValue(randomUpdate.term, randomUpdate.docIDUpto);
    }
  }
  buffer.finish();
  DocValuesUpdate.NumericDocValuesUpdate lastUpdate = randomUpdate;
  boolean termsSorted = lastUpdate.hasValue && updates.stream()
      .allMatch(update -> update.field.equals(lastUpdate.field) &&
          update.hasValue && update.getValue() == lastUpdate.getValue());
  assertBufferUpdates(buffer, updates, termsSorted);
}
 

TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) {
  this.trackAllocations = trackAllocations;
  this.nextTermsHash = nextTermsHash;
  this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter();
  intPool = new IntBlockPool(docWriter.intBlockAllocator);
  bytePool = new ByteBlockPool(docWriter.byteBlockAllocator);

  if (nextTermsHash != null) {
    // We are primary
    termBytePool = bytePool;
    nextTermsHash.termBytePool = bytePool;
  }
}
 

public void testSingleWriterReader() {
  Counter bytesUsed = Counter.newCounter();
  IntBlockPool pool = new IntBlockPool(new ByteTrackingAllocator(bytesUsed));
  
  for (int j = 0; j < 2; j++) {
    IntBlockPool.SliceWriter writer = new IntBlockPool.SliceWriter(pool);
    int start = writer.startNewSlice();
    int num = atLeast(100);
    for (int i = 0; i < num; i++) {
      writer.writeInt(i);
    }
    
    int upto = writer.getCurrentOffset();
    IntBlockPool.SliceReader reader = new IntBlockPool.SliceReader(pool);
    reader.reset(start, upto);
    for (int i = 0; i < num; i++) {
      assertEquals(i, reader.readInt());
    }
    assertTrue(reader.endOfSlice());
    if (random().nextBoolean()) {
      pool.reset(true, false);
      assertEquals(0, bytesUsed.get());
    } else {
      pool.reset(true, true);
      assertEquals(IntBlockPool.INT_BLOCK_SIZE * Integer.BYTES, bytesUsed.get());
    }
  }
}
 

FieldUpdatesBuffer(Counter bytesUsed, DocValuesUpdate.NumericDocValuesUpdate initialValue, int docUpTo) {
  this(bytesUsed, initialValue, docUpTo, true);
  if (initialValue.hasValue()) {
    numericValues = new long[] {initialValue.getValue()};
    maxNumeric = minNumeric = initialValue.getValue();
  } else {
    numericValues = new long[] {0};
  }
  bytesUsed.addAndGet(Long.BYTES);
}
 

private FieldUpdatesBuffer(Counter bytesUsed, DocValuesUpdate initialValue, int docUpTo, boolean isNumeric) {
  this.bytesUsed = bytesUsed;
  this.bytesUsed.addAndGet(SELF_SHALLOW_SIZE);
  termValues = new BytesRefArray(bytesUsed);
  termValues.append(initialValue.term.bytes);
  fields = new String[] {initialValue.term.field};
  bytesUsed.addAndGet(sizeOfString(initialValue.term.field));
  docsUpTo = new int[] {docUpTo};
  if (initialValue.hasValue == false) {
    hasValues = new FixedBitSet(1);
    bytesUsed.addAndGet(hasValues.ramBytesUsed());
  }
  this.isNumeric = isNumeric;
  byteValues = isNumeric ? null : new BytesRefArray(bytesUsed);
}
 

public SortedNumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  this.fieldInfo = fieldInfo;
  this.iwBytesUsed = iwBytesUsed;
  pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  docsWithField = new DocsWithFieldSet();
  bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed() + docsWithField.ramBytesUsed() + RamUsageEstimator.sizeOf(currentValues);
  iwBytesUsed.addAndGet(bytesUsed);
}
 

public NormValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  docsWithField = new DocsWithFieldSet();
  pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  bytesUsed = pending.ramBytesUsed() + docsWithField.ramBytesUsed();
  this.fieldInfo = fieldInfo;
  this.iwBytesUsed = iwBytesUsed;
  iwBytesUsed.addAndGet(bytesUsed);
}
 

public NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  docsWithField = new DocsWithFieldSet();
  bytesUsed = pending.ramBytesUsed() + docsWithField.ramBytesUsed();
  this.fieldInfo = fieldInfo;
  this.iwBytesUsed = iwBytesUsed;
  iwBytesUsed.addAndGet(bytesUsed);
}
 

/** streamCount: how many streams this field stores per term.
 * E.g. doc(+freq) is 1 stream, prox+offset is a second. */
TermsHashPerField(int streamCount, IntBlockPool intPool, ByteBlockPool bytePool, ByteBlockPool termBytePool,
                  Counter bytesUsed, TermsHashPerField nextPerField, String fieldName, IndexOptions indexOptions) {
  this.intPool = intPool;
  this.bytePool = bytePool;
  this.streamCount = streamCount;
  this.fieldName = fieldName;
  this.nextPerField = nextPerField;
  assert indexOptions != IndexOptions.NONE;
  this.indexOptions = indexOptions;
  PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed);
  bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
}
 

public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  this.fieldInfo = fieldInfo;
  this.iwBytesUsed = iwBytesUsed;
  hash = new BytesRefHash(
      new ByteBlockPool(
          new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)),
          BytesRefHash.DEFAULT_CAPACITY,
          new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed));
  pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  docsWithField = new DocsWithFieldSet();
  bytesUsed = pending.ramBytesUsed() + docsWithField.ramBytesUsed();
  iwBytesUsed.addAndGet(bytesUsed);
}
 

public SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  this.fieldInfo = fieldInfo;
  this.iwBytesUsed = iwBytesUsed;
  hash = new BytesRefHash(
      new ByteBlockPool(
          new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)),
          BytesRefHash.DEFAULT_CAPACITY,
          new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed));
  pending = PackedLongValues.packedBuilder(PackedInts.COMPACT);
  pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  docsWithField = new DocsWithFieldSet();
  bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed();
  iwBytesUsed.addAndGet(bytesUsed);
}
 

public BinaryDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
  this.fieldInfo = fieldInfo;
  this.bytes = new PagedBytes(BLOCK_BITS);
  this.bytesOut = bytes.getDataOutput();
  this.lengths = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);
  this.iwBytesUsed = iwBytesUsed;
  this.docsWithField = new DocsWithFieldSet();
  this.bytesUsed = lengths.ramBytesUsed() + docsWithField.ramBytesUsed();
  iwBytesUsed.addAndGet(bytesUsed);
}
 

public void testUpdateShareValues() throws IOException {
  Counter counter = Counter.newCounter();
  int intValue = random().nextInt();
  boolean valueForThree = random().nextBoolean();
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "0"), "enabled", intValue);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "1"), intValue, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "2"), intValue, Integer.MAX_VALUE);
  if (valueForThree) {
    buffer.addUpdate(new Term("id", "3"), intValue, Integer.MAX_VALUE);
  } else {
    buffer.addNoValue(new Term("id", "3"), Integer.MAX_VALUE);
  }
  buffer.addUpdate(new Term("id", "4"), intValue, Integer.MAX_VALUE);
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;
  int count = 0;
  while ((value = iterator.next()) != null) {
    boolean hasValue = count != 3 || valueForThree;
    assertEquals("" + (count++), value.termValue.utf8ToString());
    assertEquals("id", value.termField);
    assertEquals(hasValue, value.hasValue);
    if (hasValue) {
      assertEquals(intValue, value.numericValue);
    } else {
      assertEquals(0, value.numericValue);
    }
    assertEquals(Integer.MAX_VALUE, value.docUpTo);
  }
  assertTrue(buffer.isNumeric());
}
 

public void testUpdateShareValuesBinary() throws IOException {
  Counter counter = Counter.newCounter();
  boolean valueForThree = random().nextBoolean();
  DocValuesUpdate.BinaryDocValuesUpdate update =
      new DocValuesUpdate.BinaryDocValuesUpdate(new Term("id", "0"), "enabled", new BytesRef(""));
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "1"), new BytesRef(""), Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "2"), new BytesRef(""), Integer.MAX_VALUE);
  if (valueForThree) {
    buffer.addUpdate(new Term("id", "3"), new BytesRef(""), Integer.MAX_VALUE);
  } else {
    buffer.addNoValue(new Term("id", "3"), Integer.MAX_VALUE);
  }
  buffer.addUpdate(new Term("id", "4"), new BytesRef(""), Integer.MAX_VALUE);
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;
  int count = 0;
  while ((value = iterator.next()) != null) {
    boolean hasValue = count != 3 || valueForThree;
    assertEquals("" + (count++), value.termValue.utf8ToString());
    assertEquals("id", value.termField);
    assertEquals(hasValue, value.hasValue);
    if (hasValue) {
      assertEquals(new BytesRef(""), value.binaryValue);
    } else {
      assertNull(value.binaryValue);
    }
    assertEquals(Integer.MAX_VALUE, value.docUpTo);
  }
  assertFalse(buffer.isNumeric());
}
 

public void testBinaryRandom() throws IOException {
  List<DocValuesUpdate.BinaryDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  DocValuesUpdate.BinaryDocValuesUpdate randomUpdate = getRandomBinaryUpdate();
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = getRandomBinaryUpdate();
    updates.add(randomUpdate);
    if (randomUpdate.hasValue) {
      buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
    } else {
      buffer.addNoValue(randomUpdate.term, randomUpdate.docIDUpto);
    }
  }
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;

  int count = 0;
  while ((value = iterator.next()) != null) {
    randomUpdate = updates.get(count++);
    assertEquals(randomUpdate.term.bytes.utf8ToString(), value.termValue.utf8ToString());
    assertEquals(randomUpdate.term.field, value.termField);
    assertEquals("count: " + count, randomUpdate.hasValue, value.hasValue);
    if (randomUpdate.hasValue) {
      assertEquals(randomUpdate.getValue(), value.binaryValue);
    } else {
      assertNull(value.binaryValue);
    }
    assertEquals(randomUpdate.docIDUpto, value.docUpTo);
  }
  assertEquals(count, updates.size());
}
 

public void finish() throws IOException 
{
    NamedList<Object> analytics = new NamedList<>();
    rb.rsp.add("analytics", analytics);
    NamedList<Object> fieldCounts = new NamedList<>(); 
    analytics.add("mimetype()", fieldCounts);
    for (Entry<String, Counter> counter : counters.entrySet())
    {
        fieldCounts.add(counter.getKey(), counter.getValue().get());
    }
    
    if(this.delegate instanceof DelegatingCollector) {
        ((DelegatingCollector)this.delegate).finish();
    }
}
 

@Override
public void collect(int doc) throws IOException 
{
    if(sortedDocValues != null)
    {
        int ordinal = sortedDocValues.getOrd(doc);
        if(ordinal > -1)
        {
            String value = (String)schemaField.getType().toObject(schemaField, sortedDocValues.lookupOrd(ordinal));
            String group = doGroup ? mappings.get(value) : value;
            if(group == null)
            {
                group = value;
            }

            Counter counter = counters.get(group);
            if(counter == null)
            {
                counter = Counter.newCounter();
                counters.put(group, counter);
            }
            counter.addAndGet(1);
        }
    }


    leafDelegate.collect(doc);
}
 

public void testNoNumericValue() {
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", null);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(Counter.newCounter(), update, update.docIDUpto);
  assertEquals(0, buffer.getMinNumeric());
  assertEquals(0, buffer.getMaxNumeric());
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
  this.setIsPruned(in.readBoolean());
  int size = in.readInt();

  bytesUsed = Counter.newCounter();
  pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  set = new BytesRefHash(pool);

  for (long i = 0; i < size; i++) {
    set.add(in.readBytesRef());
  }
}
 

/**
 * releases a generation that was acquired by {@link #acquireTranslogGen(long)}
 */
private synchronized void releaseTranslogGen(long translogGen) {
    Counter current = translogRefCounts.get(translogGen);
    if (current == null || current.get() <= 0) {
        throw new IllegalArgumentException("translog gen [" + translogGen + "] wasn't acquired");
    }
    if (current.addAndGet(-1) == 0) {
        translogRefCounts.remove(translogGen);
    }
}
 

public IntBlockAllocator(Counter bytesUsed) {
  super(IntBlockPool.INT_BLOCK_SIZE);
  this.bytesUsed = bytesUsed;
}
 

public void testBasics() throws IOException {
  Counter counter = Counter.newCounter();
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", 6);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, 15);
  buffer.addUpdate(new Term("id", "10"), 6, 15);
  assertTrue(buffer.hasSingleValue());
  buffer.addUpdate(new Term("id", "8"), 12, 15);
  assertFalse(buffer.hasSingleValue());
  buffer.addUpdate(new Term("some_other_field", "8"), 13, 17);
  assertFalse(buffer.hasSingleValue());
  buffer.addUpdate(new Term("id", "8"), 12, 16);
  assertFalse(buffer.hasSingleValue());
  assertTrue(buffer.isNumeric());
  assertEquals(13, buffer.getMaxNumeric());
  assertEquals(6, buffer.getMinNumeric());
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("1", value.termValue.utf8ToString());
  assertEquals(6, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("10", value.termValue.utf8ToString());
  assertEquals(6, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(12, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("some_other_field", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(13, value.numericValue);
  assertEquals(17, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(12, value.numericValue);
  assertEquals(16, value.docUpTo);
  assertNull(iterator.next());
}
 

public ByteTrackingAllocator(int blockSize, Counter bytesUsed) {
  super(blockSize);
  this.bytesUsed = bytesUsed;
}
 

public BytesRefTermsSet(final CircuitBreaker breaker) {
  super(breaker);
  this.bytesUsed = Counter.newCounter();
  this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  this.set = new BytesRefHash(pool);
}
 

public ByteTrackingAllocator(Counter bytesUsed) {
  this(IntBlockPool.INT_BLOCK_SIZE, bytesUsed);
}
 

synchronized long getTranslogRefCount(long gen) {
    final Counter counter = translogRefCounts.get(gen);
    return counter == null ? 0 : counter.get();
}