Java Code Examples for org.apache.lucene.util.Counter#newCounter()

public void testNumericRandom() throws IOException {
  List<DocValuesUpdate.NumericDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  DocValuesUpdate.NumericDocValuesUpdate randomUpdate = getRandomNumericUpdate();
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = getRandomNumericUpdate();
    updates.add(randomUpdate);
    if (randomUpdate.hasValue) {
      buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
    } else {
      buffer.addNoValue(randomUpdate.term, randomUpdate.docIDUpto);
    }
  }
  buffer.finish();
  DocValuesUpdate.NumericDocValuesUpdate lastUpdate = randomUpdate;
  boolean termsSorted = lastUpdate.hasValue && updates.stream()
      .allMatch(update -> update.field.equals(lastUpdate.field) &&
          update.hasValue && update.getValue() == lastUpdate.getValue());
  assertBufferUpdates(buffer, updates, termsSorted);
}
 

public void testSortAndDedupNumericUpdatesByTerms() throws IOException {
  List<DocValuesUpdate.NumericDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  String termField = RandomPicks.randomFrom(random(), Arrays.asList("id", "_id", "some_other_field"));
  long docValue = 1 + random().nextInt(1000);
  DocValuesUpdate.NumericDocValuesUpdate randomUpdate = new DocValuesUpdate.NumericDocValuesUpdate(
      new Term(termField, Integer.toString(random().nextInt(1000))), "numeric", docValue);
  randomUpdate = randomUpdate.prepareForApply(randomDocUpTo());
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = new DocValuesUpdate.NumericDocValuesUpdate(
        new Term(termField, Integer.toString(random().nextInt(1000))), "numeric", docValue);
    randomUpdate = randomUpdate.prepareForApply(randomDocUpTo());
    updates.add(randomUpdate);
    buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
  }
  buffer.finish();
  assertBufferUpdates(buffer, updates, true);
}
 

private void readFromBytes(BytesRef bytes) {
  // Read pruned flag
  this.setIsPruned(bytes.bytes[bytes.offset++] == 1 ? true : false);

  // Read size fo the set
  int size = Bytes.readInt(bytes);

  // Read terms
  bytesUsed = Counter.newCounter();
  pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  set = new BytesRefHash(pool);

  BytesRef reusable = new BytesRef();
  for (int i = 0; i < size; i++) {
    Bytes.readBytesRef(bytes, reusable);
    set.add(reusable);
  }
}
 

@Override
public void collect(int doc) throws IOException 
{
    if(sortedDocValues != null)
    {
        int ordinal = sortedDocValues.getOrd(doc);
        if(ordinal > -1)
        {
            String value = (String)schemaField.getType().toObject(schemaField, sortedDocValues.lookupOrd(ordinal));
            String group = doGroup ? mappings.get(value) : value;
            if(group == null)
            {
                group = value;
            }

            Counter counter = counters.get(group);
            if(counter == null)
            {
                counter = Counter.newCounter();
                counters.put(group, counter);
            }
            counter.addAndGet(1);
        }
    }


    leafDelegate.collect(doc);
}
 

TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) {
  this.trackAllocations = trackAllocations;
  this.nextTermsHash = nextTermsHash;
  this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter();
  intPool = new IntBlockPool(docWriter.intBlockAllocator);
  bytePool = new ByteBlockPool(docWriter.byteBlockAllocator);

  if (nextTermsHash != null) {
    // We are primary
    termBytePool = bytePool;
    nextTermsHash.termBytePool = bytePool;
  }
}
 

public void testSingleWriterReader() {
  Counter bytesUsed = Counter.newCounter();
  IntBlockPool pool = new IntBlockPool(new ByteTrackingAllocator(bytesUsed));
  
  for (int j = 0; j < 2; j++) {
    IntBlockPool.SliceWriter writer = new IntBlockPool.SliceWriter(pool);
    int start = writer.startNewSlice();
    int num = atLeast(100);
    for (int i = 0; i < num; i++) {
      writer.writeInt(i);
    }
    
    int upto = writer.getCurrentOffset();
    IntBlockPool.SliceReader reader = new IntBlockPool.SliceReader(pool);
    reader.reset(start, upto);
    for (int i = 0; i < num; i++) {
      assertEquals(i, reader.readInt());
    }
    assertTrue(reader.endOfSlice());
    if (random().nextBoolean()) {
      pool.reset(true, false);
      assertEquals(0, bytesUsed.get());
    } else {
      pool.reset(true, true);
      assertEquals(IntBlockPool.INT_BLOCK_SIZE * Integer.BYTES, bytesUsed.get());
    }
  }
}
 

public void testUpdateShareValues() throws IOException {
  Counter counter = Counter.newCounter();
  int intValue = random().nextInt();
  boolean valueForThree = random().nextBoolean();
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "0"), "enabled", intValue);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "1"), intValue, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "2"), intValue, Integer.MAX_VALUE);
  if (valueForThree) {
    buffer.addUpdate(new Term("id", "3"), intValue, Integer.MAX_VALUE);
  } else {
    buffer.addNoValue(new Term("id", "3"), Integer.MAX_VALUE);
  }
  buffer.addUpdate(new Term("id", "4"), intValue, Integer.MAX_VALUE);
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;
  int count = 0;
  while ((value = iterator.next()) != null) {
    boolean hasValue = count != 3 || valueForThree;
    assertEquals("" + (count++), value.termValue.utf8ToString());
    assertEquals("id", value.termField);
    assertEquals(hasValue, value.hasValue);
    if (hasValue) {
      assertEquals(intValue, value.numericValue);
    } else {
      assertEquals(0, value.numericValue);
    }
    assertEquals(Integer.MAX_VALUE, value.docUpTo);
  }
  assertTrue(buffer.isNumeric());
}
 

public void testUpdateShareValuesBinary() throws IOException {
  Counter counter = Counter.newCounter();
  boolean valueForThree = random().nextBoolean();
  DocValuesUpdate.BinaryDocValuesUpdate update =
      new DocValuesUpdate.BinaryDocValuesUpdate(new Term("id", "0"), "enabled", new BytesRef(""));
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "1"), new BytesRef(""), Integer.MAX_VALUE);
  buffer.addUpdate(new Term("id", "2"), new BytesRef(""), Integer.MAX_VALUE);
  if (valueForThree) {
    buffer.addUpdate(new Term("id", "3"), new BytesRef(""), Integer.MAX_VALUE);
  } else {
    buffer.addNoValue(new Term("id", "3"), Integer.MAX_VALUE);
  }
  buffer.addUpdate(new Term("id", "4"), new BytesRef(""), Integer.MAX_VALUE);
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;
  int count = 0;
  while ((value = iterator.next()) != null) {
    boolean hasValue = count != 3 || valueForThree;
    assertEquals("" + (count++), value.termValue.utf8ToString());
    assertEquals("id", value.termField);
    assertEquals(hasValue, value.hasValue);
    if (hasValue) {
      assertEquals(new BytesRef(""), value.binaryValue);
    } else {
      assertNull(value.binaryValue);
    }
    assertEquals(Integer.MAX_VALUE, value.docUpTo);
  }
  assertFalse(buffer.isNumeric());
}
 

public void testBinaryRandom() throws IOException {
  List<DocValuesUpdate.BinaryDocValuesUpdate> updates = new ArrayList<>();
  int numUpdates = 1 + random().nextInt(1000);
  Counter counter = Counter.newCounter();
  DocValuesUpdate.BinaryDocValuesUpdate randomUpdate = getRandomBinaryUpdate();
  updates.add(randomUpdate);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, randomUpdate, randomUpdate.docIDUpto);
  for (int i = 0; i < numUpdates; i++) {
    randomUpdate = getRandomBinaryUpdate();
    updates.add(randomUpdate);
    if (randomUpdate.hasValue) {
      buffer.addUpdate(randomUpdate.term, randomUpdate.getValue(), randomUpdate.docIDUpto);
    } else {
      buffer.addNoValue(randomUpdate.term, randomUpdate.docIDUpto);
    }
  }
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value;

  int count = 0;
  while ((value = iterator.next()) != null) {
    randomUpdate = updates.get(count++);
    assertEquals(randomUpdate.term.bytes.utf8ToString(), value.termValue.utf8ToString());
    assertEquals(randomUpdate.term.field, value.termField);
    assertEquals("count: " + count, randomUpdate.hasValue, value.hasValue);
    if (randomUpdate.hasValue) {
      assertEquals(randomUpdate.getValue(), value.binaryValue);
    } else {
      assertNull(value.binaryValue);
    }
    assertEquals(randomUpdate.docIDUpto, value.docUpTo);
  }
  assertEquals(count, updates.size());
}
 

public void testNoNumericValue() {
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", null);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(Counter.newCounter(), update, update.docIDUpto);
  assertEquals(0, buffer.getMinNumeric());
  assertEquals(0, buffer.getMaxNumeric());
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
  this.setIsPruned(in.readBoolean());
  int size = in.readInt();

  bytesUsed = Counter.newCounter();
  pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  set = new BytesRefHash(pool);

  for (long i = 0; i < size; i++) {
    set.add(in.readBytesRef());
  }
}
 

/**
 * initializes searcher with a document set
 */
@Override
public void setUp() throws Exception {
  super.setUp();
  counter = Counter.newCounter(true);
  counterThread = new TimerThread(counter);
  counterThread.start();
  final String docText[] = {
      "docThatNeverMatchesSoWeCanRequireLastDocCollectedToBeGreaterThanZero",
      "one blah three",
      "one foo three multiOne",
      "one foobar three multiThree",
      "blueberry pancakes",
      "blueberry pie",
      "blueberry strudel",
      "blueberry pizza",
  };
  directory = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
  
  for (int i=0; i<N_DOCS; i++) {
    add(docText[i%docText.length], iw);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);

  BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
  booleanQuery.add(new TermQuery(new Term(FIELD_NAME, "one")), BooleanClause.Occur.SHOULD);
  // start from 1, so that the 0th doc never matches
  for (int i = 1; i < docText.length; i++) {
    String[] docTextParts = docText[i].split("\\s+");
    for (String docTextPart : docTextParts) { // large query so that search will be longer
      booleanQuery.add(new TermQuery(new Term(FIELD_NAME, docTextPart)), BooleanClause.Occur.SHOULD);
    }
  }

  query = booleanQuery.build();
  
  // warm the searcher
  searcher.search(query, 1000);
}
 

public void testBasics() throws IOException {
  Counter counter = Counter.newCounter();
  DocValuesUpdate.NumericDocValuesUpdate update =
      new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", 6);
  FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(counter, update, 15);
  buffer.addUpdate(new Term("id", "10"), 6, 15);
  assertTrue(buffer.hasSingleValue());
  buffer.addUpdate(new Term("id", "8"), 12, 15);
  assertFalse(buffer.hasSingleValue());
  buffer.addUpdate(new Term("some_other_field", "8"), 13, 17);
  assertFalse(buffer.hasSingleValue());
  buffer.addUpdate(new Term("id", "8"), 12, 16);
  assertFalse(buffer.hasSingleValue());
  assertTrue(buffer.isNumeric());
  assertEquals(13, buffer.getMaxNumeric());
  assertEquals(6, buffer.getMinNumeric());
  buffer.finish();
  FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
  FieldUpdatesBuffer.BufferedUpdate value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("1", value.termValue.utf8ToString());
  assertEquals(6, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("10", value.termValue.utf8ToString());
  assertEquals(6, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(12, value.numericValue);
  assertEquals(15, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("some_other_field", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(13, value.numericValue);
  assertEquals(17, value.docUpTo);

  value = iterator.next();
  assertNotNull(value);
  assertEquals("id", value.termField);
  assertEquals("8", value.termValue.utf8ToString());
  assertEquals(12, value.numericValue);
  assertEquals(16, value.docUpTo);
  assertNull(iterator.next());
}
 

public BytesRefTermsSet(final CircuitBreaker breaker) {
  super(breaker);
  this.bytesUsed = Counter.newCounter();
  this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
  this.set = new BytesRefHash(pool);
}