Java Code Examples for org.apache.lucene.index.IndexWriter#addDocuments()

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
  final IndexWriter indexWriter = new IndexWriter(directory, config);
  for (int i = 0; i < AMOUNT_OF_SEGMENTS; i++) {
    List<Document> segmentDocs = createDocsForSegment(i);
    indexWriter.addDocuments(segmentDocs);
    indexWriter.commit();
  }
  indexReader = DirectoryReader.open(indexWriter);
  indexWriter.close();
  indexSearcher = new IndexSearcher(indexReader);
  parentsFilter = new QueryBitSetProducer(new WildcardQuery(new Term("parent", "*")));
}
 

public void testSharedCountCollectorManager() throws Exception {
  Query q = new MatchAllDocsQuery();
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  TopDocsCollector<ScoreDoc> collector = doSearchWithThreshold( 5, 10, q, reader);
  TopDocs tdc = doConcurrentSearchWithThreshold(5, 10, q, reader);
  TopDocs tdc2 = collector.topDocs();

  CheckHits.checkEqual(q, tdc.scoreDocs, tdc2.scoreDocs);

  reader.close();
  dir.close();
}
 

private void allowDuplicateUpdate(AddUpdateCommand cmd) throws IOException {
  RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
  try {
    IndexWriter writer = iw.get();
    Iterable<Document> nestedDocs = cmd.getLuceneDocsIfNested();
    if (nestedDocs != null) {
      writer.addDocuments(nestedDocs);
    } else {
      writer.addDocument(cmd.getLuceneDocument());
    }
    if (ulog != null) ulog.add(cmd);

  } finally {
    iw.decref();
  }

}
 

private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
    numDocAppends.inc(docs.size());
}
 

public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
  Directory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)),
      newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1)))));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)),
      newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1)))));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)),
      newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)), newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2)))));
  writer.close();
  return directory;
}
 

private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException {
  Directory directory = new RAMDirectory();
  String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
  Map<String, String[]> columns = new HashMap<String, String[]>();
  for (int i = 0; i < columnFamilies.length; i++) {
    columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
  }
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
  for (int i = 0; i < numberOfDocs; i++) {
    writer.addDocuments(generatSuperDoc(random, columns, sampler));
  }
  writer.close();
  return directory;
}
 

private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
    numDocAppends.inc(docs.size());
}
 

public void testEmptyChildFilter() throws Exception {
  final Directory dir = newDirectory();
  final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
  config.setMergePolicy(NoMergePolicy.INSTANCE);
  // we don't want to merge - since we rely on certain segment setup
  final IndexWriter w = new IndexWriter(dir, config);

  final List<Document> docs = new ArrayList<>();

  docs.add(makeJob("java", 2007));
  docs.add(makeJob("python", 2010));
  docs.add(makeResume("Lisa", "United Kingdom"));
  w.addDocuments(docs);

  docs.clear();
  docs.add(makeJob("ruby", 2005));
  docs.add(makeJob("java", 2006));
  docs.add(makeResume("Frank", "United States"));
  w.addDocuments(docs);
  w.commit();

  IndexReader r = DirectoryReader.open(w);
  w.close();
  IndexSearcher s = newSearcher(r);
  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
  CheckJoinIndex.check(r, parentsFilter);

  BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
  childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
  childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST));

  ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);

  BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
  fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
  fullQuery.add(new BooleanClause(new MatchAllDocsQuery(), Occur.MUST));
  TopDocs topDocs = s.search(fullQuery.build(), 2);
  assertEquals(2, topDocs.totalHits.value);
  assertEquals(asSet("Lisa", "Frank"),
      asSet(s.doc(topDocs.scoreDocs[0].doc).get("name"), s.doc(topDocs.scoreDocs[1].doc).get("name")));

  ParentChildrenBlockJoinQuery childrenQuery =
      new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[0].doc);
  TopDocs matchingChildren = s.search(childrenQuery, 1);
  assertEquals(1, matchingChildren.totalHits.value);
  assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));

  childrenQuery = new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[1].doc);
  matchingChildren = s.search(childrenQuery, 1);
  assertEquals(1, matchingChildren.totalHits.value);
  assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));

  r.close();
  dir.close();
}
 

public void testSetMinCompetitiveScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 2);
  ScoreAndDoc scorer = new ScoreAndDoc();

  LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
  leafCollector.setScorer(scorer);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 1;
  scorer.score = 2;
  leafCollector.collect(1);
  assertNull(scorer.minCompetitiveScore);
  
  scorer.doc = 2;
  scorer.score = 3;
  leafCollector.collect(2);
  assertEquals(Math.nextUp(2f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 3;
  scorer.score = 0.5f;
  // Make sure we do not call setMinCompetitiveScore for non-competitive hits
  scorer.minCompetitiveScore = Float.NaN;
  leafCollector.collect(3);
  assertTrue(Float.isNaN(scorer.minCompetitiveScore));

  scorer.doc = 4;
  scorer.score = 4;
  leafCollector.collect(4);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  // Make sure the min score is set on scorers on new segments
  scorer = new ScoreAndDoc();
  leafCollector = collector.getLeafCollector(reader.leaves().get(1));
  leafCollector.setScorer(scorer);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 1;
  scorer.score = 4;
  leafCollector.collect(1);
  assertEquals(Math.nextUp(4f), scorer.minCompetitiveScore, 0f);

  reader.close();
  dir.close();
}
 

public void testTotalHits() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
    TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, totalHitsThreshold);
    ScoreAndDoc scorer = new ScoreAndDoc();

    LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
    leafCollector.setScorer(scorer);

    scorer.doc = 0;
    scorer.score = 3;
    leafCollector.collect(0);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    leafCollector = collector.getLeafCollector(reader.leaves().get(1));
    leafCollector.setScorer(scorer);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    scorer.doc = 5;
    scorer.score = 4;
    leafCollector.collect(1);

    TopDocs topDocs = collector.topDocs();
    assertEquals(4, topDocs.totalHits.value);
    assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
    assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
  }

  reader.close();
  dir.close();
}
 

public void testSetMinCompetitiveScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
  TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, 2);
  ScoreAndDoc scorer = new ScoreAndDoc();

  LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
  leafCollector.setScorer(scorer);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 1;
  scorer.score = 2;
  leafCollector.collect(1);
  assertNull(scorer.minCompetitiveScore);
  
  scorer.doc = 2;
  scorer.score = 3;
  leafCollector.collect(2);
  assertEquals(2f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 3;
  scorer.score = 0.5f;
  // Make sure we do not call setMinCompetitiveScore for non-competitive hits
  scorer.minCompetitiveScore = Float.NaN;
  leafCollector.collect(3);
  assertTrue(Float.isNaN(scorer.minCompetitiveScore));

  scorer.doc = 4;
  scorer.score = 4;
  leafCollector.collect(4);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  // Make sure the min score is set on scorers on new segments
  scorer = new ScoreAndDoc();
  leafCollector = collector.getLeafCollector(reader.leaves().get(1));
  leafCollector.setScorer(scorer);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 1;
  scorer.score = 4;
  leafCollector.collect(1);
  assertEquals(4f, scorer.minCompetitiveScore, 0f);

  reader.close();
  dir.close();
}
 

public void testTotalHitsWithScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
    Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
    TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, totalHitsThreshold);
    ScoreAndDoc scorer = new ScoreAndDoc();

    LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
    leafCollector.setScorer(scorer);

    scorer.doc = 0;
    scorer.score = 3;
    leafCollector.collect(0);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    leafCollector = collector.getLeafCollector(reader.leaves().get(1));
    leafCollector.setScorer(scorer);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    scorer.doc = 5;
    scorer.score = 4;
    leafCollector.collect(1);

    TopDocs topDocs = collector.topDocs();
    assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
    assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
  }

  reader.close();
  dir.close();
}