org.apache.lucene.codecs.StoredFieldsReader Java Examples

@Override
public StoredFieldsReader getFieldsReader() {
    StoredFieldsReader fieldsReader = super.getFieldsReader();
    return new FilterStoredFieldsReader(fieldsReader) {
        @Override
        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
            if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
                super.visitDocument(docID, visitor);
            } else {
                super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
                    @Override
                    public Status needsField(FieldInfo fieldInfo) throws IOException {
                        if (recoverySourceField.equals(fieldInfo.name)) {
                            return Status.NO;
                        }
                        return super.needsField(fieldInfo);
                    }
                });
            }
        }
    };
}
 

@Override
public StoredFieldsReader clone() {
  if (in == null) {
    throw new AlreadyClosedException("this FieldsReader is closed");
  }
  return new SimpleTextStoredFieldsReader(offsets, in.clone(), fieldInfos);
}
 

AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
 

private void estimateStoredFields(Map<String, Object> result) throws IOException {
  log.info("- estimating stored fields...");
  Map<String, Map<String, Object>> stats = new HashMap<>();
  for (LeafReaderContext context : reader.leaves()) {
    LeafReader leafReader = context.reader();
    EstimatingVisitor visitor = new EstimatingVisitor(stats, topN, maxLength, samplingStep);
    Bits liveDocs = leafReader.getLiveDocs();
    if (leafReader instanceof CodecReader) {
      CodecReader codecReader = (CodecReader)leafReader;
      StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
      // this instance may be faster for a full sequential pass
      StoredFieldsReader mergeInstance = storedFieldsReader.getMergeInstance();
      for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        mergeInstance.visitDocument(docId, visitor);
      }
      if (mergeInstance != storedFieldsReader) {
        mergeInstance.close();
      }
    } else {
      for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        leafReader.document(docId, visitor);
      }
    }
  }
  result.put(STORED_FIELDS, stats);
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  String value = si.getAttribute(MODE_KEY);
  if (value == null) {
    throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
  }
  Mode mode = Mode.valueOf(value);
  return impl(mode).fieldsReader(directory, si, fn, context);
}
 

private static StoredFieldsReader readerToStoredFieldsReader(final LeafReader reader) {
  return new StoredFieldsReader() {
    @Override
    public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
      reader.document(docID, visitor);
    }

    @Override
    public StoredFieldsReader clone() {
      return readerToStoredFieldsReader(reader);
    }

    @Override
    public void checkIntegrity() throws IOException {
      // We already checkIntegrity the entire reader up front
    }

    @Override
    public void close() {
    }

    @Override
    public long ramBytesUsed() {
      return 0;
    }
  };
}
 

MergeReaderWrapper(CodecReader in) throws IOException {
  this.in = in;
  
  FieldsProducer fields = in.getPostingsReader();
  if (fields != null) {
    fields = fields.getMergeInstance();
  }
  this.fields = fields;
  
  NormsProducer norms = in.getNormsReader();
  if (norms != null) {
    norms = norms.getMergeInstance();
  }
  this.norms = norms;
  
  DocValuesProducer docValues = in.getDocValuesReader();
  if (docValues != null) {
    docValues = docValues.getMergeInstance();
  }
  this.docValues = docValues;
  
  StoredFieldsReader store = in.getFieldsReader();
  if (store != null) {
    store = store.getMergeInstance();
  }
  this.store = store;
  
  TermVectorsReader vectors = in.getTermVectorsReader();
  if (vectors != null) {
    vectors = vectors.getMergeInstance();
  }
  this.vectors = vectors;
}
 

@Override
void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
  super.flush(state, sortMap);
  if (sortMap == null) {
    // we're lucky the index is already sorted, just rename the temporary file and return
    for (Map.Entry<String, String> entry : tmpDirectory.getTemporaryFiles().entrySet()) {
      tmpDirectory.rename(entry.getValue(), entry.getKey());
    }
    return;
  }
  StoredFieldsReader reader = docWriter.codec.storedFieldsFormat()
      .fieldsReader(tmpDirectory, state.segmentInfo, state.fieldInfos, IOContext.DEFAULT);
  StoredFieldsReader mergeReader = reader.getMergeInstance();
  StoredFieldsWriter sortWriter = docWriter.codec.storedFieldsFormat()
      .fieldsWriter(state.directory, state.segmentInfo, IOContext.DEFAULT);
  try {
    reader.checkIntegrity();
    CopyVisitor visitor = new CopyVisitor(sortWriter);
    for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
      sortWriter.startDocument();
      mergeReader.visitDocument(sortMap.newToOld(docID), visitor);
      sortWriter.finishDocument();
    }
    sortWriter.finish(state.fieldInfos, state.segmentInfo.maxDoc());
  } finally {
    IOUtils.close(reader, sortWriter);
    IOUtils.deleteFiles(tmpDirectory,
        tmpDirectory.getTemporaryFiles().values());
  }
}
 

@Override
public StoredFieldsReader clone() {
    return fieldsReader.clone();
}
 

FilterStoredFieldsReader(StoredFieldsReader fieldsReader) {
    this.fieldsReader = fieldsReader;
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
    throws IOException {
  CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si));
  return new CompressingStoredFieldsReader(directory, si, SEGMENT_SUFFIX, fn, context, FORMAT_NAME, compressionMode);
}
 

@Test
public void testEstimator() throws Exception {
  JettySolrRunner jetty = cluster.getRandomJetty(random());
  String randomCoreName = jetty.getCoreContainer().getAllCoreNames().iterator().next();
  SolrCore core = jetty.getCoreContainer().getCore(randomCoreName);
  RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
  try {
    SolrIndexSearcher searcher = searcherRef.get();
    // limit the max length
    IndexSizeEstimator estimator = new IndexSizeEstimator(searcher.getRawReader(), 20, 50, true, true);
    IndexSizeEstimator.Estimate estimate = estimator.estimate();
    Map<String, Long> fieldsBySize = estimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", fieldsBySize.isEmpty());
    assertEquals(fieldsBySize.toString(), fields.size(), fieldsBySize.size());
    fieldsBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Long> typesBySize = estimate.getTypesBySize();
    assertFalse("empty typesBySize", typesBySize.isEmpty());
    assertTrue("expected at least 8 types: " + typesBySize.toString(), typesBySize.size() >= 8);
    typesBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Object> summary = estimate.getSummary();
    assertNotNull("summary", summary);
    assertFalse("empty summary", summary.isEmpty());
    assertEquals(summary.keySet().toString(), fields.size(), summary.keySet().size());
    Map<String, Object> details = estimate.getDetails();
    assertNotNull("details", details);
    assertFalse("empty details", details.isEmpty());
    // by type
    assertEquals(details.keySet().toString(), 6, details.keySet().size());

    // check sampling
    estimator.setSamplingThreshold(searcher.getRawReader().maxDoc() / 2);
    IndexSizeEstimator.Estimate sampledEstimate = estimator.estimate();
    Map<String, Long> sampledFieldsBySize = sampledEstimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", sampledFieldsBySize.isEmpty());
    // verify that the sampled values are within 50% of the original values
    fieldsBySize.forEach((field, size) -> {
      Long sampledSize = sampledFieldsBySize.get(field);
      assertNotNull("sampled size for " + field + " is missing in " + sampledFieldsBySize, sampledSize);
      double delta = (double) size * 0.5;
      assertEquals("sampled size of " + field + " is wildly off", (double)size, (double)sampledSize, delta);
    });
    // verify the reader is still usable - SOLR-13694
    IndexReader reader = searcher.getRawReader();
    for (LeafReaderContext context : reader.leaves()) {
      LeafReader leafReader = context.reader();
      assertTrue("unexpected LeafReader class: " + leafReader.getClass().getName(), leafReader instanceof CodecReader);
      Bits liveDocs = leafReader.getLiveDocs();
      CodecReader codecReader = (CodecReader) leafReader;
      StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
      StoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      assertNotNull(storedFieldsReader);
      for (int docId = 0; docId < leafReader.maxDoc(); docId++) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        storedFieldsReader.visitDocument(docId, visitor);
      }
    }
  } finally {
    searcherRef.decref();
    core.close();
  }
}
 

@Override
protected StoredFieldsReader initialValue() {
  return fieldsReaderOrig.clone();
}
 

/**
 * Returns a {@code CodecReader} view of reader. 
 * <p>
 * If {@code reader} is already a {@code CodecReader}, it is returned
 * directly. Otherwise, a (slow) view is returned.
 */
public static CodecReader wrap(final LeafReader reader) throws IOException {
  if (reader instanceof CodecReader) {
    return (CodecReader) reader;
  } else {
    // simulate it slowly, over the leafReader api:
    reader.checkIntegrity();
    return new CodecReader() {

      @Override
      public TermVectorsReader getTermVectorsReader() {
        reader.ensureOpen();
        return readerToTermVectorsReader(reader);
      }

      @Override
      public StoredFieldsReader getFieldsReader() {
        reader.ensureOpen();
        return readerToStoredFieldsReader(reader);
      }

      @Override
      public NormsProducer getNormsReader() {
        reader.ensureOpen();
        return readerToNormsProducer(reader);
      }

      @Override
      public DocValuesProducer getDocValuesReader() {
        reader.ensureOpen();
        return readerToDocValuesProducer(reader);
      }

      @Override
      public FieldsProducer getPostingsReader() {
        reader.ensureOpen();
        try {
          return readerToFieldsProducer(reader);
        } catch (IOException bogus) {
          throw new AssertionError(bogus);
        }
      }

      @Override
      public FieldInfos getFieldInfos() {
        return reader.getFieldInfos();
      }

      @Override
      public PointsReader getPointsReader() {
        return pointValuesToReader(reader);
      }

      @Override
      public Bits getLiveDocs() {
        return reader.getLiveDocs();
      }

      @Override
      public int numDocs() {
        return reader.numDocs();
      }

      @Override
      public int maxDoc() {
        return reader.maxDoc();
      }

      @Override
      public CacheHelper getCoreCacheHelper() {
        return reader.getCoreCacheHelper();
      }

      @Override
      public CacheHelper getReaderCacheHelper() {
        return reader.getReaderCacheHelper();
      }

      @Override
      public String toString() {
        return "SlowCodecReaderWrapper(" + reader + ")";
      }

      @Override
      public LeafMetaData getMetaData() {
        return reader.getMetaData();
      }
    };
  }
}
 

/** Sole constructor. */
MergeState(List<CodecReader> originalReaders, SegmentInfo segmentInfo, InfoStream infoStream) throws IOException {

  this.infoStream = infoStream;

  final Sort indexSort = segmentInfo.getIndexSort();
  int numReaders = originalReaders.size();
  leafDocMaps = new DocMap[numReaders];
  List<CodecReader> readers = maybeSortReaders(originalReaders, segmentInfo);

  maxDocs = new int[numReaders];
  fieldsProducers = new FieldsProducer[numReaders];
  normsProducers = new NormsProducer[numReaders];
  storedFieldsReaders = new StoredFieldsReader[numReaders];
  termVectorsReaders = new TermVectorsReader[numReaders];
  docValuesProducers = new DocValuesProducer[numReaders];
  pointsReaders = new PointsReader[numReaders];
  fieldInfos = new FieldInfos[numReaders];
  liveDocs = new Bits[numReaders];

  int numDocs = 0;
  for(int i=0;i<numReaders;i++) {
    final CodecReader reader = readers.get(i);

    maxDocs[i] = reader.maxDoc();
    liveDocs[i] = reader.getLiveDocs();
    fieldInfos[i] = reader.getFieldInfos();

    normsProducers[i] = reader.getNormsReader();
    if (normsProducers[i] != null) {
      normsProducers[i] = normsProducers[i].getMergeInstance();
    }
    
    docValuesProducers[i] = reader.getDocValuesReader();
    if (docValuesProducers[i] != null) {
      docValuesProducers[i] = docValuesProducers[i].getMergeInstance();
    }
    
    storedFieldsReaders[i] = reader.getFieldsReader();
    if (storedFieldsReaders[i] != null) {
      storedFieldsReaders[i] = storedFieldsReaders[i].getMergeInstance();
    }
    
    termVectorsReaders[i] = reader.getTermVectorsReader();
    if (termVectorsReaders[i] != null) {
      termVectorsReaders[i] = termVectorsReaders[i].getMergeInstance();
    }
    
    fieldsProducers[i] = reader.getPostingsReader().getMergeInstance();
    pointsReaders[i] = reader.getPointsReader();
    if (pointsReaders[i] != null) {
      pointsReaders[i] = pointsReaders[i].getMergeInstance();
    }
    numDocs += reader.numDocs();
  }

  segmentInfo.setMaxDoc(numDocs);

  this.segmentInfo = segmentInfo;
  this.docMaps = buildDocMaps(readers, indexSort);
}
 

@Override
public StoredFieldsReader getFieldsReader() {
  ensureOpen();
  return core.fieldsReaderLocal.get();
}
 

@Override
public StoredFieldsReader getFieldsReader() {
  return in.getFieldsReader();
}
 

/**
 * Test stored fields.
 * @lucene.experimental
 */
public static Status.StoredFieldStatus testStoredFields(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.StoredFieldStatus status = new Status.StoredFieldStatus();

  try {
    if (infoStream != null) {
      infoStream.print("    test: stored fields.......");
    }

    // Scan stored fields for all documents
    final Bits liveDocs = reader.getLiveDocs();
    StoredFieldsReader storedFields = reader.getFieldsReader().getMergeInstance();
    for (int j = 0; j < reader.maxDoc(); ++j) {
      // Intentionally pull even deleted documents to
      // make sure they too are not corrupt:
      DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      storedFields.visitDocument(j, visitor);
      Document doc = visitor.getDocument();
      if (liveDocs == null || liveDocs.get(j)) {
        status.docCount++;
        status.totFields += doc.getFields().size();
      }
    }      

    // Validate docCount
    if (status.docCount != reader.numDocs()) {
      throw new RuntimeException("docCount=" + status.docCount + " but saw " + status.docCount + " undeleted docs");
    }

    msg(infoStream, String.format(Locale.ROOT, "OK [%d total field count; avg %.1f fields per doc] [took %.3f sec]",
                                  status.totFields,
                                  (((float) status.totFields)/status.docCount),
                                  nsToSec(System.nanoTime() - startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
    FieldInfos fn, IOContext context) throws IOException {
  return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, 
      context, formatName, compressionMode);
}
 

@Override
public StoredFieldsReader getMergeInstance() {
  ensureOpen();
  return new CompressingStoredFieldsReader(this, true);
}
 

@Override
public StoredFieldsReader clone() {
  ensureOpen();
  return new CompressingStoredFieldsReader(this, false);
}
 

@Override
public StoredFieldsReader getFieldsReader() {
  return fieldsReader.get();
}
 

@Override
protected StoredFieldsReader initialValue() {
  return in.getFieldsReader().getMergeInstance();
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  return delegate.fieldsReader(directory, si, fn, context);
}
 

@Override
public StoredFieldsReader getMergeInstance() {
  return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc, true);
}
 

@Override
public StoredFieldsReader clone() {
  assert merging == false : "Merge instances do not support cloning";
  return new AssertingStoredFieldsReader(in.clone(), maxDoc, false);
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc(), false);
}
 

@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {;
  return new SimpleTextStoredFieldsReader(directory, si, fn, context);
}
 

/** 
 * Expert: retrieve thread-private StoredFieldsReader
 * @lucene.internal 
 */
public abstract StoredFieldsReader getFieldsReader();