Java Code Examples for org.apache.lucene.search.IndexSearcher#setQueryCache()

static CodecReader wrapReader(String recoverySourceField, CodecReader reader, Supplier<Query> retainSourceQuerySupplier)
    throws IOException {
    NumericDocValues recoverySource = reader.getNumericDocValues(recoverySourceField);
    if (recoverySource == null || recoverySource.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
        return reader; // early terminate - nothing to do here since non of the docs has a recovery source anymore.
    }
    IndexSearcher s = new IndexSearcher(reader);
    s.setQueryCache(null);
    Weight weight = s.createWeight(s.rewrite(retainSourceQuerySupplier.get()), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    Scorer scorer = weight.scorer(reader.getContext());
    if (scorer != null) {
        BitSet recoverySourceToKeep = BitSet.of(scorer.iterator(), reader.maxDoc());
        // calculating the cardinality is significantly cheaper than skipping all bulk-merging we might do
        // if retentions are high we keep most of it
        if (recoverySourceToKeep.cardinality() == reader.maxDoc()) {
            return reader; // keep all source
        }
        return new SourcePruningFilterCodecReader(recoverySourceField, reader, recoverySourceToKeep);
    } else {
        return new SourcePruningFilterCodecReader(recoverySourceField, reader, null);
    }
}
 

private int loadQueries(IndexShard shard) {
    shard.refresh("percolator_load_queries");
    // NOTE: we acquire the searcher via the engine directly here since this is executed right
    // before the shard is marked as POST_RECOVERY
    try (Engine.Searcher searcher = shard.engine().acquireSearcher("percolator_load_queries")) {
        Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME));
        QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService);
        IndexSearcher indexSearcher = new IndexSearcher(searcher.reader());
        indexSearcher.setQueryCache(null);
        indexSearcher.search(query, queryCollector);
        Map<BytesRef, Query> queries = queryCollector.queries();
        for (Map.Entry<BytesRef, Query> entry : queries.entrySet()) {
            Query previousQuery = percolateQueries.put(entry.getKey(), entry.getValue());
            shardPercolateService.addedQuery(entry.getKey(), previousQuery, entry.getValue());
        }
        return queries.size();
    } catch (Exception e) {
        throw new PercolatorException(shardId.index(), "failed to load queries from percolator index", e);
    }
}
 

/**
 * Set the index reader. Takes ownership of that index reader, that is,
 * internally performs indexReader.incRef() (If caller no longer needs that 
 * reader it should decRef()/close() it after calling this method, otherwise, 
 * the reader will remain open). 
 * @param indexReader The indexReader to set.
 */
public synchronized void setIndexReader(DirectoryReader indexReader) throws IOException {
  if (indexReader == this.indexReader) {
    return;
  }
  
  if (this.indexReader != null) {
    // Release current IR
    this.indexReader.decRef();
  }

  this.indexReader = indexReader;
  if (indexReader != null) {
    // Hold reference to new IR
    indexReader.incRef();
    indexSearcher = new IndexSearcher(indexReader);
    // TODO Some day we should make the query cache in this module configurable and control clearing the cache
    indexSearcher.setQueryCache(null);
  } else {
    indexSearcher = null;
  }
}
 

private void createIndex(IndexWriterConfig config, Directory target, DirectoryReader reader, Query preserveFilter, boolean negateFilter) throws IOException {
  boolean success = false;
  final IndexWriter w = new IndexWriter(target, config);
  try {
    final IndexSearcher searcher = new IndexSearcher(reader);
    searcher.setQueryCache(null);
    preserveFilter = searcher.rewrite(preserveFilter);
    final Weight preserveWeight = searcher.createWeight(preserveFilter, ScoreMode.COMPLETE_NO_SCORES, 1);
    final List<LeafReaderContext> leaves = reader.leaves();
    final CodecReader[] subReaders = new CodecReader[leaves.size()];
    int i = 0;
    for (final LeafReaderContext ctx : leaves) {
      subReaders[i++] = new DocumentFilteredLeafIndexReader(ctx, preserveWeight, negateFilter);
    }
    w.addIndexes(subReaders);
    success = true;
  } finally {
    if (success) {
      w.close();
    } else {
      IOUtils.closeWhileHandlingException(w);
    }
  }
}
 

/**
 * As part of loading the segment in ImmutableSegmentLoader,
 * we load the text index (per column if it exists) and store
 * the reference in {@link org.apache.pinot.core.segment.index.column.PhysicalColumnIndexContainer}
 * similar to how it is done for other types of indexes.
 * @param column column name
 * @param indexDir segment index directory
 * @param numDocs number of documents in the segment
 */
public LuceneTextIndexReader(String column, File indexDir, int numDocs,
    @Nullable Map<String, String> textIndexProperties) {
  _column = column;
  try {
    File indexFile = getTextIndexFile(indexDir);
    _indexDirectory = FSDirectory.open(indexFile.toPath());
    _indexReader = DirectoryReader.open(_indexDirectory);
    _indexSearcher = new IndexSearcher(_indexReader);
    if (textIndexProperties == null || !Boolean.parseBoolean(textIndexProperties.get(FieldConfig.TEXT_INDEX_ENABLE_QUERY_CACHE))) {
      // Disable Lucene query result cache. While it helps a lot with performance for
      // repeated queries, on the downside it cause heap issues.
      _indexSearcher.setQueryCache(null);
    }
    // TODO: consider using a threshold of num docs per segment to decide between building
    // mapping file upfront on segment load v/s on-the-fly during query processing
    _docIdTranslator = new DocIdTranslator(indexDir, _column, numDocs, _indexSearcher);
    _standardAnalyzer = new StandardAnalyzer(LuceneTextIndexCreator.ENGLISH_STOP_WORDS_SET);
  } catch (Exception e) {
    LOGGER
        .error("Failed to instantiate Lucene text index reader for column {}, exception {}", column, e.getMessage());
    throw new RuntimeException(e);
  }
}
 

/**
 * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher
 * gets wrapped and a new {@link Engine.Searcher} instances is returned, otherwise the provided {@link Engine.Searcher} is returned.
 *
 * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search)
 */
public final Engine.Searcher wrap(Engine.Searcher engineSearcher) throws IOException {
    final ElasticsearchDirectoryReader elasticsearchDirectoryReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader());
    if (elasticsearchDirectoryReader == null) {
        throw new IllegalStateException("Can't wrap non elasticsearch directory reader");
    }
    NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader());
    DirectoryReader reader = wrap(nonClosingReaderWrapper);
    if (reader != nonClosingReaderWrapper) {
        if (reader.getReaderCacheHelper() != elasticsearchDirectoryReader.getReaderCacheHelper()) {
            throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey, wrappers must override this method and delegate" +
                    " to the original readers core cache key. Wrapped readers can't be used as cache keys since their are used only per request which would lead to subtle bugs");
        }
        if (ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader) != elasticsearchDirectoryReader) {
            // prevent that somebody wraps with a non-filter reader
            throw new IllegalStateException("wrapped directory reader hides actual ElasticsearchDirectoryReader but shouldn't");
        }
    }

    final IndexSearcher origIndexSearcher = engineSearcher.searcher();
    final IndexSearcher innerIndexSearcher = new IndexSearcher(reader);
    innerIndexSearcher.setQueryCache(origIndexSearcher.getQueryCache());
    innerIndexSearcher.setQueryCachingPolicy(origIndexSearcher.getQueryCachingPolicy());
    innerIndexSearcher.setSimilarity(origIndexSearcher.getSimilarity());
    // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point
    // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten
    // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times
    final IndexSearcher indexSearcher = wrap(innerIndexSearcher);
    if (reader == nonClosingReaderWrapper && indexSearcher == innerIndexSearcher) {
        return engineSearcher;
    } else {
        // we close the reader to make sure wrappers can release resources if needed....
        // our NonClosingReaderWrapper makes sure that our reader is not closed
        return new Engine.Searcher(engineSearcher.source(), indexSearcher, () ->
            IOUtils.close(indexSearcher.getIndexReader(), // this will close the wrappers excluding the NonClosingReaderWrapper
            engineSearcher)); // this will run the closeable on the wrapped engine searcher
    }
}
 

@Override
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
    IndexSearcher searcher = super.newSearcher(reader, previousReader);
    searcher.setQueryCache(engineConfig.getQueryCache());
    searcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy());
    searcher.setSimilarity(engineConfig.getSimilarity());
    return searcher;
}
 

@Override
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
    IndexSearcher searcher = super.newSearcher(reader, previousReader);
    searcher.setQueryCache(engineConfig.getQueryCache());
    searcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy());
    return searcher;
}
 

/**
 * Creates and returns a searcher that can be used to execute arbitrary
 * Lucene queries and to collect the resulting query results as hits.
 * 
 * @return a searcher
 */
public IndexSearcher createSearcher() {
  MemoryIndexReader reader = new MemoryIndexReader();
  IndexSearcher searcher = new IndexSearcher(reader); // ensures no auto-close !!
  searcher.setSimilarity(normSimilarity);
  searcher.setQueryCache(null);
  return searcher;
}
 

@Override
public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
  IndexSearcher searcher = super.newSearcher(reader, previousReader);
  searcher.setQueryCache(null);
  termFilters.put(reader.getReaderCacheHelper().getKey(), new QueryTermFilter(reader));
  reader.getReaderCacheHelper().addClosedListener(termFilters::remove);
  return searcher;
}
 

/**
 * Match a DocumentBatch against the queries stored in the Monitor, also returning information
 * about which queries were selected by the presearcher, and why.
 *
 * @param docs    a DocumentBatch to match against the index
 * @param factory a {@link MatcherFactory} to use to create a {@link CandidateMatcher} for the match run
 * @param <T>     the type of QueryMatch produced by the CandidateMatcher
 * @return a {@link PresearcherMatches} object containing debug information
 * @throws IOException on IO errors
 */
public <T extends QueryMatch> PresearcherMatches<T> debug(Document[] docs, MatcherFactory<T> factory)
    throws IOException {
  try (DocumentBatch batch = DocumentBatch.of(analyzer, docs)) {
    LeafReader reader = batch.get();
    IndexSearcher searcher = new IndexSearcher(reader);
    searcher.setQueryCache(null);
    PresearcherQueryCollector<T> collector = new PresearcherQueryCollector<>(factory.createMatcher(searcher));
    long buildTime = queryIndex.search(t -> new ForceNoBulkScoringQuery(presearcher.buildQuery(reader, t)), collector);
    return collector.getMatches(buildTime);
  }
}
 

private void count(DoubleValuesSource valueSource, List<MatchingDocs> matchingDocs) throws IOException {

    DoubleRange[] ranges = (DoubleRange[]) this.ranges;

    LongRange[] longRanges = new LongRange[ranges.length];
    for(int i=0;i<ranges.length;i++) {
      DoubleRange range = ranges[i];
      longRanges[i] =  new LongRange(range.label,
                                     NumericUtils.doubleToSortableLong(range.min), true,
                                     NumericUtils.doubleToSortableLong(range.max), true);
    }

    LongRangeCounter counter = new LongRangeCounter(longRanges);

    int missingCount = 0;
    for (MatchingDocs hits : matchingDocs) {
      DoubleValues fv = valueSource.getValues(hits.context, null);
      
      totCount += hits.totalHits;
      final DocIdSetIterator fastMatchDocs;
      if (fastMatchQuery != null) {
        final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(hits.context);
        final IndexSearcher searcher = new IndexSearcher(topLevelContext);
        searcher.setQueryCache(null);
        final Weight fastMatchWeight = searcher.createWeight(searcher.rewrite(fastMatchQuery), ScoreMode.COMPLETE_NO_SCORES, 1);
        Scorer s = fastMatchWeight.scorer(hits.context);
        if (s == null) {
          continue;
        }
        fastMatchDocs = s.iterator();
      } else {
        fastMatchDocs = null;
      }

      DocIdSetIterator docs = hits.bits.iterator();

      for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; ) {
        if (fastMatchDocs != null) {
          int fastMatchDoc = fastMatchDocs.docID();
          if (fastMatchDoc < doc) {
            fastMatchDoc = fastMatchDocs.advance(doc);
          }

          if (doc != fastMatchDoc) {
            doc = docs.advance(fastMatchDoc);
            continue;
          }
        }
        // Skip missing docs:
        if (fv.advanceExact(doc)) {
          counter.add(NumericUtils.doubleToSortableLong(fv.doubleValue()));
        } else {
          missingCount++;
        }

        doc = docs.nextDoc();
      }
    }

    missingCount += counter.fillCounts(counts);
    totCount -= missingCount;
  }
 

private static Scorer getScorer(Query query, CodecReader reader) throws IOException {
  IndexSearcher s = new IndexSearcher(reader);
  s.setQueryCache(null);
  Weight weight = s.createWeight(s.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
  return weight.scorer(reader.getContext());
}
 

private void count(LongValuesSource valueSource, List<MatchingDocs> matchingDocs) throws IOException {

    LongRange[] ranges = (LongRange[]) this.ranges;

    LongRangeCounter counter = new LongRangeCounter(ranges);

    int missingCount = 0;
    for (MatchingDocs hits : matchingDocs) {
      LongValues fv = valueSource.getValues(hits.context, null);
      
      totCount += hits.totalHits;
      final DocIdSetIterator fastMatchDocs;
      if (fastMatchQuery != null) {
        final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(hits.context);
        final IndexSearcher searcher = new IndexSearcher(topLevelContext);
        searcher.setQueryCache(null);
        final Weight fastMatchWeight = searcher.createWeight(searcher.rewrite(fastMatchQuery), ScoreMode.COMPLETE_NO_SCORES, 1);
        Scorer s = fastMatchWeight.scorer(hits.context);
        if (s == null) {
          continue;
        }
        fastMatchDocs = s.iterator();
      } else {
        fastMatchDocs = null;
      }

      DocIdSetIterator docs = hits.bits.iterator();      
      for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; ) {
        if (fastMatchDocs != null) {
          int fastMatchDoc = fastMatchDocs.docID();
          if (fastMatchDoc < doc) {
            fastMatchDoc = fastMatchDocs.advance(doc);
          }

          if (doc != fastMatchDoc) {
            doc = docs.advance(fastMatchDoc);
            continue;
          }
        }
        // Skip missing docs:
        if (fv.advanceExact(doc)) {
          counter.add(fv.longValue());
        } else {
          missingCount++;
        }

        doc = docs.nextDoc();
      }
    }
    
    int x = counter.fillCounts(counts);

    missingCount += x;

    //System.out.println("totCount " + totCount + " x " + x + " missingCount " + missingCount);
    totCount -= missingCount;
  }
 

@Override
public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
    // Reset parentFilter, so we resolve the parentDocs for each new segment being searched
    this.parentFilter = null;
    final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx);
    final IndexSearcher searcher = new IndexSearcher(topLevelContext);
    searcher.setQueryCache(null);
    final Weight weight = searcher.createNormalizedWeight(childFilter, false);
    Scorer childDocsScorer = weight.scorer(ctx);
    if (childDocsScorer == null) {
        childDocs = null;
    } else {
        childDocs = childDocsScorer.iterator();
    }

    return new LeafBucketCollectorBase(sub, null) {
        @Override
        public void collect(int parentDoc, long bucket) throws IOException {
            // here we translate the parent doc to a list of its nested docs, and then call super.collect for evey one of them so they'll be collected

            // if parentDoc is 0 then this means that this parent doesn't have child docs (b/c these appear always before the parent doc), so we can skip:
            if (parentDoc == 0 || childDocs == null) {
                return;
            }
            if (parentFilter == null) {
                // The aggs are instantiated in reverse, first the most inner nested aggs and lastly the top level aggs
                // So at the time a nested 'nested' aggs is parsed its closest parent nested aggs hasn't been constructed.
                // So the trick is to set at the last moment just before needed and we can use its child filter as the
                // parent filter.

                // Additional NOTE: Before this logic was performed in the setNextReader(...) method, but the the assumption
                // that aggs instances are constructed in reverse doesn't hold when buckets are constructed lazily during
                // aggs execution
                Query parentFilterNotCached = findClosestNestedPath(parent());
                if (parentFilterNotCached == null) {
                    parentFilterNotCached = Queries.newNonNestedFilter();
                }
                parentFilter = context.searchContext().bitsetFilterCache().getBitSetProducer(parentFilterNotCached);
                parentDocs = parentFilter.getBitSet(ctx);
                if (parentDocs == null) {
                    // There are no parentDocs in the segment, so return and set childDocs to null, so we exit early for future invocations.
                    childDocs = null;
                    return;
                }
            }

            final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
            int childDocId = childDocs.docID();
            if (childDocId <= prevParentDoc) {
                childDocId = childDocs.advance(prevParentDoc + 1);
            }

            for (; childDocId < parentDoc; childDocId = childDocs.nextDoc()) {
                collectBucket(sub, childDocId, bucket);
            }
        }
    };
}
 

public FilterableTermsEnum(IndexReader reader, String field, int docsEnumFlag, @Nullable Query filter) throws IOException {
    if ((docsEnumFlag != PostingsEnum.FREQS) && (docsEnumFlag != PostingsEnum.NONE)) {
        throw new IllegalArgumentException("invalid docsEnumFlag of " + docsEnumFlag);
    }
    this.docsEnumFlag = docsEnumFlag;
    if (filter == null) {
        // Important - need to use the doc count that includes deleted docs
        // or we have this issue: https://github.com/elasticsearch/elasticsearch/issues/7951
        numDocs = reader.maxDoc();
    }
    List<LeafReaderContext> leaves = reader.leaves();
    List<Holder> enums = new ArrayList<>(leaves.size());
    final Weight weight;
    if (filter == null) {
        weight = null;
    } else {
        final IndexSearcher searcher = new IndexSearcher(reader);
        searcher.setQueryCache(null);
        weight = searcher.createNormalizedWeight(filter, false);
    }
    for (LeafReaderContext context : leaves) {
        Terms terms = context.reader().terms(field);
        if (terms == null) {
            continue;
        }
        TermsEnum termsEnum = terms.iterator();
        if (termsEnum == null) {
            continue;
        }
        BitSet bits = null;
        if (weight != null) {
            Scorer scorer = weight.scorer(context);
            if (scorer == null) {
                // fully filtered, none matching, no need to iterate on this
                continue;
            }
            DocIdSetIterator docs = scorer.iterator();

            // we want to force apply deleted docs
            final Bits liveDocs = context.reader().getLiveDocs();
            if (liveDocs != null) {
                docs = new FilteredDocIdSetIterator(docs) {
                    @Override
                    protected boolean match(int doc) {
                        return liveDocs.get(doc);
                    }
                };
            }

            BitDocIdSet.Builder builder = new BitDocIdSet.Builder(context.reader().maxDoc());
            builder.or(docs);
            bits = builder.build().bits();

            // Count how many docs are in our filtered set
            // TODO make this lazy-loaded only for those that need it?
            numDocs += bits.cardinality();
        }
        enums.add(new Holder(termsEnum, bits));
    }
    this.enums = enums.toArray(new Holder[enums.size()]);
}
 

/**
 * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher
 * gets wrapped and a new {@link Engine.Searcher} instances is returned, otherwise the provided {@link Engine.Searcher} is returned.
 *
 * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search)
 */
public final Engine.Searcher wrap(EngineConfig engineConfig, final Engine.Searcher engineSearcher) throws IOException {
    final ElasticsearchDirectoryReader elasticsearchDirectoryReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader());
    if (elasticsearchDirectoryReader == null) {
        throw new IllegalStateException("Can't wrap non elasticsearch directory reader");
    }
    if (wrapper == null) {
        return engineSearcher;
    }
    NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader());
    DirectoryReader reader = wrapper.wrap(nonClosingReaderWrapper);
    if (reader != nonClosingReaderWrapper) {
        if (reader.getCoreCacheKey() != elasticsearchDirectoryReader.getCoreCacheKey()) {
            throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey, wrappers must override this method and delegate" +
                    " to the original readers core cache key. Wrapped readers can't be used as cache keys since their are used only per request which would lead to subtle bugs");
        }
        if (ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader) != elasticsearchDirectoryReader) {
            // prevent that somebody wraps with a non-filter reader
            throw new IllegalStateException("wrapped directory reader hides actual ElasticsearchDirectoryReader but shouldn't");
        }
    }

    final IndexSearcher innerIndexSearcher = new IndexSearcher(reader);
    innerIndexSearcher.setQueryCache(engineConfig.getQueryCache());
    innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy());
    innerIndexSearcher.setSimilarity(engineConfig.getSimilarity());
    // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point
    // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten
    // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times
    final IndexSearcher indexSearcher = wrapper.wrap(engineConfig, innerIndexSearcher);
    if (reader == nonClosingReaderWrapper && indexSearcher == innerIndexSearcher) {
        return engineSearcher;
    } else {
        return new Engine.Searcher(engineSearcher.source(), indexSearcher) {
            @Override
            public void close() throws ElasticsearchException {
                try {
                    reader().close();
                    // we close the reader to make sure wrappers can release resources if needed....
                    // our NonClosingReaderWrapper makes sure that our reader is not closed
                } catch (IOException e) {
                    throw new ElasticsearchException("failed to close reader", e);
                } finally {
                    engineSearcher.close();
                }

            }
        };
    }
}
 

public LeafIndexLookup(LeafReaderContext ctx) {
    reader = ctx.reader();
    parentReader = ReaderUtil.getTopLevelContext(ctx).reader();
    indexSearcher = new IndexSearcher(parentReader);
    indexSearcher.setQueryCache(null);
}