org.apache.lucene.document.BinaryDocValuesField Java Examples

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    XContentParser parser = context.parser();
    if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
        // we are in the parse Phase
        String id = parser.text();
        if (context.id() != null && !context.id().equals(id)) {
            throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
        }
        context.id(id);
    } // else we are in the pre/post parse phase

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        fields.add(new Field(fieldType().names().indexName(), context.id(), fieldType()));
    }
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(fieldType().names().indexName(), new BytesRef(context.id())));
    }
}
 

private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 

public void testSameFieldNameForPostingAndDocValue() throws Exception {
  // LUCENE-5192: FieldInfos.Builder neglected to update
  // globalFieldNumbers.docValuesType map if the field existed, resulting in
  // potentially adding the same field with different DV types.
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new NumericDocValuesField("f", 5));
  writer.addDocument(doc);
  writer.commit();
  
  Document doc2 = new Document();
  doc2.add(new BinaryDocValuesField("f", new BytesRef("mock")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(doc2);
  });
  writer.rollback();
  
  dir.close();
}
 

public void testMixedTypesAfterReopenAppend3() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  // Also add another document so there is a segment to write here:
  w2.addDocument(new Document());
  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 

public void testMixedTypesAfterReopenAppend2() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 

public void testAddBinaryTwice() throws IOException {
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
  iwriter.addDocument(doc);
  
  doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
  expectThrows(IllegalArgumentException.class, () -> {
    iwriter.addDocument(doc);
  });
  
  IndexReader ir = iwriter.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  
  iwriter.close();
  directory.close();
}
 

public void testDifferentTypedDocValuesField() throws Exception {
  Directory d = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), d);
  Document doc = new Document();
  doc.add(new NumericDocValuesField("field", 17));
  w.addDocument(doc);
  
  // Index doc values are single-valued so we should not
  // be able to add same field more than once:
  doc.add(new BinaryDocValuesField("field", new BytesRef("blah")));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });

  DirectoryReader r = w.getReader();
  w.close();
  NumericDocValues values = DocValues.getNumeric(getOnlyLeafReader(r), "field");
  assertEquals(0, values.nextDoc());
  assertEquals(17, values.longValue());
  r.close();
  d.close();
}
 

public void testUpdateTwoNonexistingTerms() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  writer.addDocument(doc);
  // update w/ multiple nonexisting terms in same field
  writer.updateBinaryDocValue(new Term("c", "foo"), "f1", toBytes(2L));
  writer.updateBinaryDocValue(new Term("c", "bar"), "f1", toBytes(2L));
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(1L, getValue(bdv));
  reader.close();
  
  dir.close();
}
 

public void testUpdateAllDeletedSegment() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.commit();
  writer.deleteDocuments(new Term("id", "doc")); // delete all docs in the first segment
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("id", "doc"), "f1", toBytes(2L));
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(2L, getValue(bdv));
  reader.close();
  
  dir.close();
}
 

public void testUpdateBinaryDVFieldWithSameNameAsPostingField() throws Exception {
  // this used to fail because FieldInfos.Builder neglected to update
  // globalFieldMaps.docValuesTypes map
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new BinaryDocValuesField("f", toBytes(5L)));
  writer.addDocument(doc);
  writer.commit();
  writer.updateBinaryDocValue(new Term("f", "mock-value"), "f", toBytes(17L));
  writer.close();
  
  DirectoryReader r = DirectoryReader.open(dir);
  BinaryDocValues bdv = r.leaves().get(0).reader().getBinaryDocValues("f");
  assertEquals(0, bdv.nextDoc());
  assertEquals(17, getValue(bdv));
  r.close();
  
  dir.close();
}
 

public void testUpdateSameDocMultipleTimes() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("key", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateBinaryDocValue(new Term("key", "doc"), "bdv", toBytes(17L)); // update existing field
  writer.updateBinaryDocValue(new Term("key", "doc"), "bdv", toBytes(3L)); // update existing field 2nd time in this commit
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(3, getValue(bdv));
  }
  reader.close();
  dir.close();
}
 

public void testUpdateNotExistingFieldDV() throws IOException {
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
    Document doc = new Document();
    doc.add(new StringField("id", "1", Store.YES));
    doc.add(new NumericDocValuesField("test", 1));
    writer.addDocument(doc);
    if (random().nextBoolean()) {
      writer.commit();
    }
    writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));

    Document doc1 = new Document();
    doc1.add(new StringField("id", "2", Store.YES));
    doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
        writer.addDocument(doc1)
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());

    iae = expectThrows(IllegalArgumentException.class, () ->
        writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
  }
}
 

private List<Indexable> buildIndexables(List<MonitorQuery> updates) {
  List<Indexable> indexables = new ArrayList<>();
  for (MonitorQuery mq : updates) {
    if (serializer != null && mq.getQueryString() == null) {
      throw new IllegalArgumentException("Cannot add a MonitorQuery with a null string representation to a non-ephemeral Monitor");
    }
    BytesRef serialized = serializer == null ? EMPTY : serializer.serialize(mq);
    for (QueryCacheEntry qce : QueryCacheEntry.decompose(mq, decomposer)) {
      Document doc = presearcher.indexQuery(qce.matchQuery, mq.getMetadata());
      doc.add(new StringField(FIELDS.query_id, qce.queryId, Field.Store.NO));
      doc.add(new SortedDocValuesField(FIELDS.cache_id, new BytesRef(qce.cacheId)));
      doc.add(new SortedDocValuesField(FIELDS.query_id, new BytesRef(qce.queryId)));
      doc.add(new BinaryDocValuesField(FIELDS.mq, serialized));
      indexables.add(new Indexable(qce, doc));
    }
  }
  return indexables;
}
 

@Override
protected void addRandomFields(Document doc) {
  if (usually()) {
    doc.add(new NumericDocValuesField("ndv", random().nextInt(1 << 12)));
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
    doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  int numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
  }
}
 

@Override
public Field[] createIndexableFields(Shape shape) {
  int bufSize = Math.max(128, (int) (this.indexLastBufSize * 1.5));//50% headroom over last
  ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize);
  final BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes
  try {
    ctx.getBinaryCodec().writeShape(new DataOutputStream(byteStream), shape);
    //this is a hack to avoid redundant byte array copying by byteStream.toByteArray()
    byteStream.writeTo(new FilterOutputStream(null/*not used*/) {
      @Override
      public void write(byte[] b, int off, int len) throws IOException {
        bytesRef.bytes = b;
        bytesRef.offset = off;
        bytesRef.length = len;
      }
    });
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  this.indexLastBufSize = bytesRef.length;//cache heuristic
  return new Field[]{new BinaryDocValuesField(getFieldName(), bytesRef)};
}
 

@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random()));

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv_binary", new BytesRef("lucene")));
  doc.add(new SortedDocValuesField("dv_sorted", new BytesRef("abc")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("python")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("java")));
  doc.add(new NumericDocValuesField("dv_numeric", 42L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 22L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 11L));
  doc.add(newStringField("no_dv", "aaa", Field.Store.NO));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 

private void buildIndexWithFacets(Directory indexDir, Directory taxoDir, boolean asc) throws IOException {
  IndexWriterConfig config = newIndexWriterConfig(null);
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexDir, config);
  
  DirectoryTaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
  for (int i = 1; i <= NUM_DOCS; i++) {
    Document doc = new Document();
    for (int j = i; j <= NUM_DOCS; j++) {
      int facetValue = asc ? j: NUM_DOCS - j;
      doc.add(new FacetField("tag", Integer.toString(facetValue)));
    }
    // add a facet under default dim config
    doc.add(new FacetField("id", Integer.toString(i)));
    
    // make sure OrdinalMappingLeafReader ignores non-facet BinaryDocValues fields
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
    doc.add(new BinaryDocValuesField("cbdv", new BytesRef(Integer.toString(i*2))));
    writer.addDocument(facetConfig.build(taxonomyWriter, doc));
  }
  taxonomyWriter.commit();
  taxonomyWriter.close();
  writer.commit();
  writer.close();
}
 

private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 

public void testDocumentWithNoValue() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
    if (i == 0) { // index only one document with value
      doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
    }
    writer.addDocument(doc);
  }
  writer.commit();
  
  // update all docs' bdv field
  writer.updateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv", toBytes(17L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  LeafReader r = reader.leaves().get(0).reader();
  BinaryDocValues bdv = r.getBinaryDocValues("bdv");
  for (int i = 0; i < r.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(17, getValue(bdv));
  }
  
  reader.close();
  dir.close();
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    Field uid = new Field(NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE);
    context.uid(uid);
    fields.add(uid);
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
    }
}
 

public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
  doc.add(new TextField("text", "quick brown fox", Field.Store.NO));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
  TermsEnum tenum = leafReader.terms("text").iterator();

  assertEquals("brown", tenum.next().utf8ToString());
  PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(1, penum.nextPosition());
  assertEquals(6, penum.startOffset());
  assertEquals(11, penum.endOffset());

  assertEquals("fox", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(2, penum.nextPosition());
  assertEquals(12, penum.startOffset());
  assertEquals(15, penum.endOffset());

  assertEquals("quick", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(0, penum.nextPosition());
  assertEquals(0, penum.startOffset());
  assertEquals(5, penum.endOffset());

  BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text");
  assertEquals(0, binaryDocValues.nextDoc());
  assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString());
}
 

public void testNormsWithDocValues() throws Exception {
  MemoryIndex mi = new MemoryIndex(true, true);
  MockAnalyzer mockAnalyzer = new MockAnalyzer(random());

  mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer);
  mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
  Field field = new TextField("text", "quick brown fox", Field.Store.NO);
  doc.add(field);
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
  writer.addDocument(doc);
  writer.close();

  IndexReader controlIndexReader = DirectoryReader.open(dir);
  LeafReader controlLeafReader =  controlIndexReader.leaves().get(0).reader();

  NumericDocValues norms = controlLeafReader.getNormValues("text");
  assertEquals(0, norms.nextDoc());
  NumericDocValues norms2 = leafReader.getNormValues("text");
  assertEquals(0, norms2.nextDoc());
  assertEquals(norms.longValue(), norms2.longValue());

  controlIndexReader.close();
  dir.close();
}
 

private void addDVs(Document d1, int i) {
  d1.add(new NumericDocValuesField("numeric", i));
  d1.add(new BinaryDocValuesField("binary", new BytesRef(""+i)));
  d1.add(new SortedDocValuesField("sorted", new BytesRef(""+i)));
  d1.add(new SortedNumericDocValuesField("sortednumeric", i));
  d1.add(new SortedSetDocValuesField("sortedset", new BytesRef(""+i)));
}
 

public void testUpdatesOrder() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("upd", "t1", Store.NO));
  doc.add(new StringField("upd", "t2", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(2L)); // update f1 to 2
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f2", toBytes(2L)); // update f2 to 2
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f1", toBytes(3L)); // update f1 to 3
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f2", toBytes(3L)); // update f2 to 3
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(4L)); // update f1 to 4 (but not f2)
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(4, getValue(bdv));
  bdv = reader.leaves().get(0).reader().getBinaryDocValues("f2");
  assertEquals(0, bdv.nextDoc());
  assertEquals(3, getValue(bdv));
  reader.close();
  
  dir.close();
}
 

public void testDeleteUnusedUpdatesFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "d0", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
  writer.addDocument(doc);

  // update each field twice to make sure all unneeded files are deleted
  for (String f : new String[] { "f1", "f2" }) {
    writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(2L));
    writer.commit();
    int numFiles = dir.listAll().length;
    
    // update again, number of files shouldn't change (old field's gen is
    // removed) 
    writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(3L));
    writer.commit();
    
    assertEquals(numFiles, dir.listAll().length);
  }

  writer.close();
  dir.close();
}
 

public void testUpdateDocumentByMultipleTerms() throws Exception {
  // make sure the order of updates is respected, even when multiple terms affect same document
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("k1", "v1", Store.NO));
  doc.add(new StringField("k2", "v2", Store.NO));
  doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateBinaryDocValue(new Term("k1", "v1"), "bdv", toBytes(17L));
  writer.updateBinaryDocValue(new Term("k2", "v2"), "bdv", toBytes(3L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(3, getValue(bdv));
  }
  reader.close();
  dir.close();
}
 

public void testMultipleBinaryDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setMaxBufferedDocs(10); // prevent merges
  IndexWriter writer = new IndexWriter(dir, conf);
  
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
    doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));
    doc.add(new BinaryDocValuesField("bdv2", toBytes(i)));
    writer.addDocument(doc);
  }
  writer.commit();
  
  // update all docs' bdv1 field
  writer.updateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv1", toBytes(17L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  LeafReader r = reader.leaves().get(0).reader();
  
  BinaryDocValues bdv1 = r.getBinaryDocValues("bdv1");
  BinaryDocValues bdv2 = r.getBinaryDocValues("bdv2");
  for (int i = 0; i < r.maxDoc(); i++) {
    assertEquals(i, bdv1.nextDoc());
    assertEquals(17, getValue(bdv1));
    assertEquals(i, bdv2.nextDoc());
    assertEquals(i, getValue(bdv2));
  }
  
  reader.close();
  dir.close();
}
 

public void testAddNullNumericDocValues() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  if (random().nextBoolean()) {
    doc.add(new NumericDocValuesField("foo", null));
  } else {
    doc.add(new BinaryDocValuesField("foo", null));
  }
  IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> iw.addDocument(doc));
  assertEquals("field=\"foo\": null value not allowed", iae.getMessage());
  IOUtils.close(iw, dir);
}
 

/**
 * field with binary docvalues
 */
public void testBinaryField() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("foo", new BytesRef("bar")));
  iw.addDocument(doc);
  DirectoryReader dr = DirectoryReader.open(iw);
  LeafReader r = getOnlyLeafReader(dr);
  
  // ok
  assertNotNull(DocValues.getBinary(r, "foo"));
  
  // errors
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getNumeric(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSorted(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSortedSet(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSortedNumeric(r, "foo");
  });
  
  dr.close();
  iw.close();
  dir.close();
}
 

public void testDocValuesIntegration() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(null);
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
  doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
  doc.add(new NumericDocValuesField("numeric", 42));
  doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
  doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
  iw.addDocument(doc);
  DirectoryReader ir = iw.getReader();
  iw.close();
  LeafReader ar = getOnlyLeafReader(ir);
  
  // Binary type: can be retrieved via getTerms()
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER);
  });
  
  // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER);
  });
  
  // Numeric type: can be retrieved via getInts() and so on
  NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER);
  assertEquals(0, numeric.nextDoc());
  assertEquals(42, numeric.longValue());
     
  // SortedSet type: can be retrieved via getDocTermOrds() 
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER);
  });
  
  ir.close();
  dir.close();
}