org.apache.lucene.codecs.DocValuesProducer Java Examples

private RefCount<DocValuesProducer> newDocValuesProducer(SegmentCommitInfo si, Directory dir, final Long gen, FieldInfos infos) throws IOException {
  Directory dvDir = dir;
  String segmentSuffix = "";
  if (gen.longValue() != -1) {
    dvDir = si.info.dir; // gen'd files are written outside CFS, so use SegInfo directory
    segmentSuffix = Long.toString(gen.longValue(), Character.MAX_RADIX);
  }

  // set SegmentReadState to list only the fields that are relevant to that gen
  SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, IOContext.READ, segmentSuffix);
  DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat();
  return new RefCount<DocValuesProducer>(dvFormat.fieldsProducer(srs)) {
    @SuppressWarnings("synthetic-access")
    @Override
    protected void release() throws IOException {
      object.close();
      synchronized (SegmentDocValues.this) {
        genDVProducers.remove(gen);
      }
    }
  };
}
 

/**
 * init most recent DocValues for the current commit
 */
private DocValuesProducer initDocValuesProducer() throws IOException {

  if (fieldInfos.hasDocValues() == false) {
    return null;
  } else {
    Directory dir;
    if (core.cfsReader != null) {
      dir = core.cfsReader;
    } else {
      dir = si.info.dir;
    }
    if (si.hasFieldUpdates()) {
      return new SegmentDocValuesProducer(si, dir, core.coreFieldInfos, fieldInfos, segDocValues);
    } else {
      // simple case, no DocValues updates
      return segDocValues.getDocValuesProducer(-1L, si, dir, fieldInfos);
    }
  }
}
 

@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);

  long valueCount = 0;
  int lastDocID = -1;
  while (true) {
    int docID = values.nextDoc();
    if (docID == NO_MORE_DOCS) {
      break;
    }
    assert values.docID() > lastDocID;
    lastDocID = values.docID();
    int count = values.docValueCount();
    assert count > 0;
    valueCount += count;
    long previous = Long.MIN_VALUE;
    for (int i = 0; i < count; i++) {
      long nextValue = values.nextValue();
      assert nextValue >= previous;
      previous = nextValue;
    }
  }
  in.addSortedNumericField(field, valuesProducer);
}
 

@Override
public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  BinaryDocValues values = valuesProducer.getBinary(field);
  
  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    BytesRef value = values.binaryValue();
    assert value.isValid();
  }

  in.addBinaryField(field, valuesProducer);
}
 

@Override
public DocValuesProducer getDocValuesReader() {
    DocValuesProducer producer = in.getDocValuesReader();
    // TODO: move this nullness stuff out
    if (producer == null) {
        producer = FilterDocValuesProducer.EMPTY;
    }
    return new UninvertedVersions(producer, this);
}
 

MergeReaderWrapper(CodecReader in) throws IOException {
  this.in = in;
  
  FieldsProducer fields = in.getPostingsReader();
  if (fields != null) {
    fields = fields.getMergeInstance();
  }
  this.fields = fields;
  
  NormsProducer norms = in.getNormsReader();
  if (norms != null) {
    norms = norms.getMergeInstance();
  }
  this.norms = norms;
  
  DocValuesProducer docValues = in.getDocValuesReader();
  if (docValues != null) {
    docValues = docValues.getMergeInstance();
  }
  this.docValues = docValues;
  
  StoredFieldsReader store = in.getFieldsReader();
  if (store != null) {
    store = store.getMergeInstance();
  }
  this.store = store;
  
  TermVectorsReader vectors = in.getTermVectorsReader();
  if (vectors != null) {
    vectors = vectors.getMergeInstance();
  }
  this.vectors = vectors;
}
 

@Override
public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedSetField()");
  }
  delegate.addSortedSetField(field, valuesProducer);
}
 

@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedNumericField()");
  }
  delegate.addSortedNumericField(field, valuesProducer);
}
 

@Override
public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedField()");
  }
  delegate.addSortedField(field, valuesProducer);
}
 

@Override
public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addBinaryField()");
  }
  delegate.addBinaryField(field, valuesProducer);
}
 

@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addNumericField()");
  }
  delegate.addNumericField(field, valuesProducer);
}
 

/** Returns the {@link DocValuesProducer} for the given generation. */
synchronized DocValuesProducer getDocValuesProducer(long gen, SegmentCommitInfo si, Directory dir, FieldInfos infos) throws IOException {
  RefCount<DocValuesProducer> dvp = genDVProducers.get(gen);
  if (dvp == null) {
    dvp = newDocValuesProducer(si, dir, gen, infos);
    assert dvp != null;
    genDVProducers.put(gen, dvp);
  } else {
    dvp.incRef();
  }
  return dvp.get();
}
 

private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, int maxDoc, PrintStream infoStream, DocValuesStatus status) throws Exception {
  switch(fi.getDocValuesType()) {
    case SORTED:
      status.totalSortedFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSorted);
      checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
      checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
      break;
    case SORTED_NUMERIC:
      status.totalSortedNumericFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSortedNumeric);
      checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi), dvReader.getSortedNumeric(fi));
      break;
    case SORTED_SET:
      status.totalSortedSetFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSortedSet);
      checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi), dvReader.getSortedSet(fi));
      break;
    case BINARY:
      status.totalBinaryFields++;
      checkDVIterator(fi, maxDoc, dvReader::getBinary);
      checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi), dvReader.getBinary(fi));
      break;
    case NUMERIC:
      status.totalNumericFields++;
      checkDVIterator(fi, maxDoc, dvReader::getNumeric);
      checkNumericDocValues(fi.name, dvReader.getNumeric(fi), dvReader.getNumeric(fi));
      break;
    default:
      throw new AssertionError();
  }
}
 

AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {
  this.in = in;
  this.maxDoc = maxDoc;
  this.merging = merging;
  this.creationThread = Thread.currentThread();
  // do a few simple checks on init
  assert toString() != null;
  assert ramBytesUsed() >= 0;
  assert getChildResources() != null;
}
 

@Override
public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  SortedDocValues values = valuesProducer.getSorted(field);

  int valueCount = values.getValueCount();
  assert valueCount <= maxDoc;
  BytesRef lastValue = null;
  for (int ord=0;ord<valueCount;ord++) {
    BytesRef b = values.lookupOrd(ord);
    assert b != null;
    assert b.isValid();
    if (ord > 0) {
      assert b.compareTo(lastValue) > 0;
    }
    lastValue = BytesRef.deepCopyOf(b);
  }
  
  FixedBitSet seenOrds = new FixedBitSet(valueCount);
  
  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    int ord = values.ordValue();
    assert ord >= 0 && ord < valueCount;
    seenOrds.set(ord);
  }
  
  assert seenOrds.cardinality() == valueCount;
  in.addSortedField(field, valuesProducer);
}
 

@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  NumericDocValues values = valuesProducer.getNumeric(field);

  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    long value = values.longValue();
  }
  
  in.addNumericField(field, valuesProducer);
}
 

@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
  assert state.fieldInfos.hasDocValues();
  DocValuesProducer producer = in.fieldsProducer(state);
  assert producer != null;
  return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);
}
 

@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  meta.writeInt(field.number);
  meta.writeByte(Lucene80DocValuesFormat.SORTED_NUMERIC);

  long[] stats = writeValues(field, valuesProducer);
  int numDocsWithField = Math.toIntExact(stats[0]);
  long numValues = stats[1];
  assert numValues >= numDocsWithField;

  meta.writeInt(numDocsWithField);
  if (numValues > numDocsWithField) {
    long start = data.getFilePointer();
    meta.writeLong(start);
    meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);

    final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT);
    long addr = 0;
    addressesWriter.add(addr);
    SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
    for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
      addr += values.docValueCount();
      addressesWriter.add(addr);
    }
    addressesWriter.finish();
    meta.writeLong(data.getFilePointer() - start);
  }
}
 

@Override
public long ramBytesUsed() {
  long size = 0;
  for (Map.Entry<String,DocValuesProducer> entry : formats.entrySet()) {
    size += (entry.getKey().length() * Character.BYTES) + entry.getValue().ramBytesUsed();
  }
  return size;
}
 

@Override
public DocValuesProducer getDocValuesReader() {
    DocValuesProducer docValuesReader = super.getDocValuesReader();
    return new FilterDocValuesProducer(docValuesReader) {
        @Override
        public NumericDocValues getNumeric(FieldInfo field) throws IOException {
            NumericDocValues numeric = super.getNumeric(field);
            if (recoverySourceField.equals(field.name)) {
                assert numeric != null : recoverySourceField + " must have numeric DV but was null";
                final DocIdSetIterator intersection;
                if (recoverySourceToKeep == null) {
                    // we can't return null here lucenes DocIdMerger expects an instance
                    intersection = DocIdSetIterator.empty();
                } else {
                    intersection = ConjunctionDISI.intersectIterators(Arrays.asList(numeric,
                                                                                    new BitSetIterator(recoverySourceToKeep, recoverySourceToKeep.length())));
                }
                return new FilterNumericDocValues(numeric) {
                    @Override
                    public int nextDoc() throws IOException {
                        return intersection.nextDoc();
                    }

                    @Override
                    public int advance(int target) {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public boolean advanceExact(int target) {
                        throw new UnsupportedOperationException();
                    }
                };

            }
            return numeric;
        }
    };
}
 

/**
 * Test docvalues.
 * @lucene.experimental
 */
public static Status.DocValuesStatus testDocValues(CodecReader reader,
                                                   PrintStream infoStream,
                                                   boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.DocValuesStatus status = new Status.DocValuesStatus();
  try {
    if (infoStream != null) {
      infoStream.print("    test: docvalues...........");
    }
    DocValuesProducer dvReader = reader.getDocValuesReader();
    if (dvReader != null) {
      dvReader = dvReader.getMergeInstance();
    }
    for (FieldInfo fieldInfo : reader.getFieldInfos()) {
      if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
        status.totalValueFields++;
        checkDocValues(fieldInfo, dvReader, reader.maxDoc(), infoStream, status);
      }
    }

    msg(infoStream, String.format(Locale.ROOT,
                                  "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET] [took %.3f sec]",
                                  status.totalValueFields,
                                  status.totalBinaryFields,
                                  status.totalNumericFields,
                                  status.totalSortedFields,
                                  status.totalSortedNumericFields,
                                  status.totalSortedSetFields,
                                  nsToSec(System.nanoTime()-startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }
  return status;
}
 

@Override
public final DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
  return new FieldsReader(state);
}
 

@Override
public DocValuesProducer getMergeInstance() {
  return new FieldsReader(this);
}
 

@Override
public void checkIntegrity() throws IOException {
  for (DocValuesProducer format : formats.values()) {
    format.checkIntegrity();
  }
}
 

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSortedSet(field);
}
 

@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSortedNumeric(field);
}
 

FilterDocValuesProducer(DocValuesProducer in) {
    this.in = in;
}
 

@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSorted(field);
}
 

@Override
public DocValuesProducer getDocValuesReader() {
  return docValuesProducer;
}
 

public OrdScoreCollector(int maxDoc,
                         int segments,
                         DocValuesProducer collapseValuesProducer,
                         int nullPolicy,
                         IntIntHashMap boostDocsMap,
                         IndexSearcher searcher) throws IOException {
  this.maxDoc = maxDoc;
  this.contexts = new LeafReaderContext[segments];
  List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
  for(int i=0; i<con.size(); i++) {
    contexts[i] = con.get(i);
  }

  this.collapsedSet = new FixedBitSet(maxDoc);
  this.collapseValuesProducer = collapseValuesProducer;
  this.collapseValues = collapseValuesProducer.getSorted(null);

  int valueCount = collapseValues.getValueCount();
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  this.ords = new IntIntDynamicMap(valueCount, -1);
  this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
  this.nullPolicy = nullPolicy;
  if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
    nullScores = new FloatArrayList();
  }

  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostOrds = new IntArrayList();
    this.boostDocs = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
  }
}