Java Code Examples for org.apache.lucene.index.LeafReader#getBinaryDocValues()

public void testIndexingPointsAndDocValues() throws Exception {
  FieldType type = new FieldType();
  type.setDimensions(1, 4);
  type.setDocValuesType(DocValuesType.BINARY);
  type.freeze();
  Document doc = new Document();
  byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8);
  doc.add(new BinaryPoint("field", packedPoint, type));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  assertEquals(1, leafReader.getPointValues("field").size());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMinPackedValue());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMaxPackedValue());

  BinaryDocValues dvs = leafReader.getBinaryDocValues("field");
  assertEquals(0, dvs.nextDoc());
  assertEquals("term", dvs.binaryValue().utf8ToString());
}
 

public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
  if (valuesIn == null) {
    valuesIn = reader.getSortedDocValues(field);
  }

  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  }

  final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info == null) {
    return DocValues.emptyBinary();
  } else if (info.getDocValuesType() != DocValuesType.NONE) {
    throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
  } else if (info.getIndexOptions() == IndexOptions.NONE) {
    return DocValues.emptyBinary();
  }

  BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
  return impl.iterator();
}
 

public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
  doc.add(new TextField("text", "quick brown fox", Field.Store.NO));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
  TermsEnum tenum = leafReader.terms("text").iterator();

  assertEquals("brown", tenum.next().utf8ToString());
  PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(1, penum.nextPosition());
  assertEquals(6, penum.startOffset());
  assertEquals(11, penum.endOffset());

  assertEquals("fox", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(2, penum.nextPosition());
  assertEquals(12, penum.startOffset());
  assertEquals(15, penum.endOffset());

  assertEquals("quick", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(0, penum.nextPosition());
  assertEquals(0, penum.startOffset());
  assertEquals(5, penum.endOffset());

  BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text");
  assertEquals(0, binaryDocValues.nextDoc());
  assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString());
}
 

private void assertOrdinalsExist(String field, IndexReader ir) throws IOException {
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    if (r.getBinaryDocValues(field) != null) {
      return; // not all segments must have this DocValues
    }
  }
  fail("no ordinals found for " + field);
}
 

/**
 * Returns a {@link DocIdSetIterator} from the given field or null if the field doesn't exist
 * in the reader or if the reader has no doc values for the field.
 */
public static DocIdSetIterator getDocValuesDocIdSetIterator(String field, LeafReader reader) throws IOException {
  FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  final DocIdSetIterator iterator;
  if (fieldInfo != null) {
    switch (fieldInfo.getDocValuesType()) {
      case NONE:
        iterator = null;
        break;
      case NUMERIC:
        iterator = reader.getNumericDocValues(field);
        break;
      case BINARY:
        iterator = reader.getBinaryDocValues(field);
        break;
      case SORTED:
        iterator = reader.getSortedDocValues(field);
        break;
      case SORTED_NUMERIC:
        iterator = reader.getSortedNumericDocValues(field);
        break;
      case SORTED_SET:
        iterator = reader.getSortedSetDocValues(field);
        break;
      default:
        throw new AssertionError();
    }
    return iterator;
  }
  return null;
}
 

private BitsEntry createValueDocValues(LeafReader reader, String field) throws IOException {
  FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  
  DocValuesType dvType = fieldInfo.getDocValuesType();
  DocIdSetIterator iterator;
  switch(dvType) {
  case NUMERIC:
    iterator = reader.getNumericDocValues(field);
    break;
  case BINARY:
    iterator = reader.getBinaryDocValues(field);
    break;
  case SORTED:
    iterator = reader.getSortedDocValues(field);
    break;
  case SORTED_NUMERIC:
    iterator = reader.getSortedNumericDocValues(field);
    break;
  case SORTED_SET:
    iterator = reader.getSortedSetDocValues(field);
    break;
  default:
    throw new AssertionError();
  }

  FixedBitSet bits = new FixedBitSet(reader.maxDoc());
  while (true) {
    int docID = iterator.nextDoc();
    if (docID == DocIdSetIterator.NO_MORE_DOCS) {
      break;
    }
    bits.set(docID);
  }

  return new BitsEntry(bits);
}
 

public void testDocValues() throws Exception {
  Document doc = new Document();
  doc.add(new NumericDocValuesField("numeric", 29L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L));
  doc.add(new BinaryDocValuesField("binary", new BytesRef("a")));
  doc.add(new SortedDocValuesField("sorted", new BytesRef("b")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));

  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
  NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
  assertEquals(0, numericDocValues.nextDoc());
  assertEquals(29L, numericDocValues.longValue());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, numericDocValues.nextDoc());
  SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
  assertEquals(0, sortedNumericDocValues.nextDoc());
  assertEquals(5, sortedNumericDocValues.docValueCount());
  assertEquals(30L, sortedNumericDocValues.nextValue());
  assertEquals(31L, sortedNumericDocValues.nextValue());
  assertEquals(32L, sortedNumericDocValues.nextValue());
  assertEquals(32L, sortedNumericDocValues.nextValue());
  assertEquals(33L, sortedNumericDocValues.nextValue());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, sortedNumericDocValues.nextDoc());
  BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
  assertEquals(0, binaryDocValues.nextDoc());
  assertEquals("a", binaryDocValues.binaryValue().utf8ToString());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, binaryDocValues.nextDoc());
  SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
  assertEquals(0, sortedDocValues.nextDoc());
  assertEquals("b", sortedDocValues.binaryValue().utf8ToString());
  assertEquals(0, sortedDocValues.ordValue());
  assertEquals("b", sortedDocValues.lookupOrd(0).utf8ToString());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, sortedDocValues.nextDoc());
  SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
  assertEquals(3, sortedSetDocValues.getValueCount());
  assertEquals(0, sortedSetDocValues.nextDoc());
  assertEquals(0L, sortedSetDocValues.nextOrd());
  assertEquals(1L, sortedSetDocValues.nextOrd());
  assertEquals(2L, sortedSetDocValues.nextOrd());
  assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
  assertEquals("c", sortedSetDocValues.lookupOrd(0L).utf8ToString());
  assertEquals("d", sortedSetDocValues.lookupOrd(1L).utf8ToString());
  assertEquals("f", sortedSetDocValues.lookupOrd(2L).utf8ToString());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, sortedDocValues.nextDoc());
}
 

private static void checkReaderSanity(LeafReader reader) throws IOException {
  for (FieldInfo info : reader.getFieldInfos()) {
    
    // reader shouldn't return normValues if the field does not have them
    if (!info.hasNorms()) {
      if (reader.getNormValues(info.name) != null) {
        throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
      }
    }
    
    // reader shouldn't return docValues if the field does not have them
    // reader shouldn't return multiple docvalues types for the same field.
    switch(info.getDocValuesType()) {
      case NONE:
        if (reader.getBinaryDocValues(info.name) != null ||
            reader.getNumericDocValues(info.name) != null ||
            reader.getSortedDocValues(info.name) != null || 
            reader.getSortedSetDocValues(info.name) != null) {
          throw new RuntimeException("field: " + info.name + " has docvalues but should omit them!");
        }
        break;
      case SORTED:
        if (reader.getBinaryDocValues(info.name) != null ||
            reader.getNumericDocValues(info.name) != null ||
            reader.getSortedNumericDocValues(info.name) != null ||
            reader.getSortedSetDocValues(info.name) != null) {
          throw new RuntimeException(info.name + " returns multiple docvalues types!");
        }
        break;
      case SORTED_NUMERIC:
        if (reader.getBinaryDocValues(info.name) != null ||
            reader.getNumericDocValues(info.name) != null ||
            reader.getSortedSetDocValues(info.name) != null ||
            reader.getSortedDocValues(info.name) != null) {
          throw new RuntimeException(info.name + " returns multiple docvalues types!");
        }
        break;
      case SORTED_SET:
        if (reader.getBinaryDocValues(info.name) != null ||
            reader.getNumericDocValues(info.name) != null ||
            reader.getSortedNumericDocValues(info.name) != null ||
            reader.getSortedDocValues(info.name) != null) {
          throw new RuntimeException(info.name + " returns multiple docvalues types!");
        }
        break;
      case BINARY:
        if (reader.getNumericDocValues(info.name) != null ||
            reader.getSortedDocValues(info.name) != null ||
            reader.getSortedNumericDocValues(info.name) != null ||
            reader.getSortedSetDocValues(info.name) != null) {
          throw new RuntimeException(info.name + " returns multiple docvalues types!");
        }
        break;
      case NUMERIC:
        if (reader.getBinaryDocValues(info.name) != null ||
            reader.getSortedDocValues(info.name) != null ||
            reader.getSortedNumericDocValues(info.name) != null ||
            reader.getSortedSetDocValues(info.name) != null) {
          throw new RuntimeException(info.name + " returns multiple docvalues types!");
        }
        break;
      default:
        throw new AssertionError();
    }
  }
}
 

private BinaryRangeDocValues getValues(LeafReader reader, String field) throws IOException {
  BinaryDocValues binaryDocValues = reader.getBinaryDocValues(field);

  return new BinaryRangeDocValues(binaryDocValues, numDims, numBytesPerDimension);
}