org.apache.lucene.index.SortedSetDocValues Java Examples

/** Converts {@link SortedSetDocValues} into an {@code Iterable<BytesRef>} for all the values.
 *
 * @deprecated Consume {@link SortedSetDocValues} instead. */
@Deprecated
public static Iterable<BytesRef> valuesIterable(final SortedSetDocValues values) {
  return new Iterable<BytesRef>() {
    @Override
    public Iterator<BytesRef> iterator() {
      return new Iterator<BytesRef>() {
        private long nextOrd;
  
        @Override
        public boolean hasNext() {
          return nextOrd < values.getValueCount();
        }

        @Override
        public BytesRef next() {
          try {
            return values.lookupOrd(nextOrd++);
          } catch (IOException e) {
            throw new RuntimeException(e);
          }
        }
      };
    }
  };
}
 

private long lookupTerm(SortedSetDocValues docValues, BytesRef key, long startOrd) throws IOException {
  long low = startOrd;
  long high = docValues.getValueCount()-1;

  while (low <= high) {
    long mid = (low + high) >>> 1;
    final BytesRef term = docValues.lookupOrd(mid);
    int cmp = term.compareTo(key);

    if (cmp < 0) {
      low = mid + 1;
    } else if (cmp > 0) {
      high = mid - 1;
    } else {
      return mid; // key found
    }
  }

  return -(low + 1);  // key not found.
}
 

private void collectPerSeg(SortedSetDocValues multiDv, DocIdSetIterator disi, LongValues toGlobal) throws IOException {
  int segMax = (int)multiDv.getValueCount();
  final int[] counts = getCountArr( segMax );

  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    if (multiDv.advanceExact(doc)) {
      for(;;) {
        int segOrd = (int)multiDv.nextOrd();
        if (segOrd < 0) break;
        counts[segOrd]++;
      }
    }
  }

  for (int i=0; i<segMax; i++) {
    int segCount = counts[i];
    if (segCount > 0) {
      int slot = toGlobal == null ? (i) : (int) toGlobal.get(i);
      countAcc.incrementCount(slot, segCount);
    }
  }
}
 

@Override
public void collect(int doc) throws IOException {
  if (docValues.advanceExact(doc)) {
    long ord;
    while ((ord = docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      int termID = collectedTerms.add(docValues.lookupOrd(ord));
      if (termID < 0) {
        termID = -termID - 1;
      } else {
        if (termID >= scoreSums.length) {
          scoreSums = ArrayUtil.grow(scoreSums);
          scoreCounts = ArrayUtil.grow(scoreCounts);
        }
      }
    
      scoreSums[termID] += scorer.score();
      scoreCounts[termID]++;
    }
  }
}
 

@Override
public String value() {
    try {
        if (values.advanceExact(docId)) {
            long ord = values.nextOrd();
            if (values.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
                throw new GroupByOnArrayUnsupportedException(columnName);
            }
            BytesRef encoded = values.lookupOrd(ord);
            return (String) DocValueFormat.IP.format(encoded);
        } else {
            return null;
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

private SortedSetDocValues validateAndFetchDocValues(SolrIndexSearcher solrSearcher, String fieldName, String querySide) throws IOException {
  final IndexSchema schema = solrSearcher.getSchema();
  final SchemaField field = schema.getFieldOrNull(fieldName);
  if (field == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, querySide + " field '" + fieldName + "' does not exist");
  }

  if (!field.hasDocValues()) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "'top-level' join queries require both 'from' and 'to' fields to have docValues, but " + querySide +
            " field [" + fieldName +  "] does not.");
  }

  final LeafReader leafReader = solrSearcher.getSlowAtomicReader();
  if (field.multiValued()) {
    return DocValues.getSortedSet(leafReader, fieldName);
  }
  return DocValues.singleton(DocValues.getSorted(leafReader, fieldName));
}
 

/** accumulates per-segment multi-valued facet counts, mapping to global ordinal space on-the-fly */
static void accumMultiGeneric(int counts[], int startTermIndex, SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  final LongValues ordMap = map == null ? null : map.getGlobalOrds(subIndex);
  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    if (si.advanceExact(doc)) {
      // strange do-while to collect the missing count (first ord is NO_MORE_ORDS)
      int term = (int) si.nextOrd();
      do {
        if (map != null) {
          term = (int) ordMap.get(term);
        }
        int arrIdx = term-startTermIndex;
        if (arrIdx>=0 && arrIdx<counts.length) counts[arrIdx]++;
      } while ((term = (int) si.nextOrd()) >= 0);
    } else if (startTermIndex == -1) {
      counts[0]++; // missing count
    }
  }
}
 

private BitsetBounds convertFromOrdinalsIntoToField(LongBitSet fromOrdBitSet, SortedSetDocValues fromDocValues,
                                                    LongBitSet toOrdBitSet, SortedSetDocValues toDocValues) throws IOException {
  long fromOrdinal = 0;
  long firstToOrd = BitsetBounds.NO_MATCHES;
  long lastToOrd = 0;

  while (fromOrdinal < fromOrdBitSet.length() && (fromOrdinal = fromOrdBitSet.nextSetBit(fromOrdinal)) >= 0) {
    final BytesRef fromBytesRef = fromDocValues.lookupOrd(fromOrdinal);
    final long toOrdinal = lookupTerm(toDocValues, fromBytesRef, lastToOrd);
    if (toOrdinal >= 0) {
      toOrdBitSet.set(toOrdinal);
      if (firstToOrd == BitsetBounds.NO_MATCHES) firstToOrd = toOrdinal;
      lastToOrd = toOrdinal;
    }
    fromOrdinal++;
  }

    return new BitsetBounds(firstToOrd, lastToOrd);
}
 

@Override
public FunctionValues getValues(Map<Object, Object> context, LeafReaderContext readerContext) throws IOException {
  SortedSetDocValues sortedSet = DocValues.getSortedSet(readerContext.reader(), field);
  SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
  return new DocTermsIndexDocValues(this, view) {
    @Override
    protected String toTerm(String readableValue) {
      return readableValue;
    }

    @Override
    public Object objectVal(int doc) throws IOException {
      return strVal(doc);
    }
  };
}
 

private Optional<DocValues> createSortedSetDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedSetDocValues ssvalues = IndexUtils.getSortedSetDocvalues(reader, field);

  if (ssvalues.advanceExact(docid)) {
    List<BytesRef> values = new ArrayList<>();

    long ord;
    while ((ord = ssvalues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      values.add(BytesRef.deepCopyOf(ssvalues.lookupOrd(ord)));
    }

    DocValues dv = DocValues.of(
        dvType,
        values,
        Collections.emptyList()
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 

private long lookupTerm(SortedSetDocValues docValues, BytesRef key, long startOrd) throws IOException {
  long low = startOrd;
  long high = docValues.getValueCount()-1;

  while (low <= high) {
    long mid = (low + high) >>> 1;
    final BytesRef term = docValues.lookupOrd(mid);
    int cmp = term.compareTo(key);

    if (cmp < 0) {
      low = mid + 1;
    } else if (cmp > 0) {
      high = mid - 1;
    } else {
      return mid; // key found
    }
  }

  return -(low + 1);  // key not found.
}
 

@Override
public void collectValues(int doc, int slotNum) throws IOException {
  long newOrd = MISSING;
  if (minmax == 1) {// min
    newOrd = subDv.nextOrd();
  } else  { // max
    long ord;
    while ((ord = subDv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      newOrd = ord;
    }
  }

  long currOrd = slotOrd[slotNum];
  long finalOrd = toGlobal==null ? newOrd : toGlobal.get(newOrd);
  if (currOrd == MISSING || Long.compare(finalOrd, currOrd) * minmax < 0) {
    slotOrd[slotNum] = finalOrd;
  }
}
 

private FieldComparator<?> getStringComparator(int numHits) {
  return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {

    @Override
    protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
      SortedSetDocValues sortedSet = DocValues.getSortedSet(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptySorted();
      }
      return BlockJoinSelector.wrap(sortedSet, type, parents, toIter(children));
    }

  };
}
 

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
  SortedSetDocValues sortedSetDocValues = _sortedSetDocValuesCache.get(field.number);
  if (sortedSetDocValues != null) {
    return sortedSetDocValues;
  }
  synchronized (_sortedSetDocValuesCache) {
    sortedSetDocValues = _sortedSetDocValuesCache.get(field.number);
    if (sortedSetDocValues != null) {
      return sortedSetDocValues;
    }
    sortedSetDocValues = newSortedSetDocValues(field);
    if (_cache && sortedSetDocValues != null) {
      _sortedSetDocValuesCache.put(field.number, sortedSetDocValues);
    }
    return sortedSetDocValues;
  }
}
 

private void buildFieldMap( ResponseBuilder rb ) throws IOException {
  Log.debug( "buildFieldMap" );
  SolrIndexSearcher searcher = rb.req.getSearcher();
  // build a synonym map from the SortedDocValues -
  // for each field value: lower case, stemmed, lookup synonyms from synonyms.txt - map to fieldValue
  SynonymMap.Builder fieldBuilder = new SynonymMap.Builder( true );
  SynonymMap.Builder termBuilder = new SynonymMap.Builder( true );
    
  ArrayList<String> searchFields = getStringFields( searcher );

  for (String searchField : searchFields ) {
    Log.debug( "adding searchField " + searchField );
    CharsRef fieldChars = new CharsRef( searchField );
    SortedSetDocValues sdv = FieldCache.DEFAULT.getDocTermOrds( searcher.getAtomicReader( ), searchField );
    if (sdv == null) continue;
    Log.debug( "got SortedSetDocValues for " + searchField );
    TermsEnum te = sdv.termsEnum();
    while (te.next() != null) {
      BytesRef term = te.term();
      String fieldValue = term.utf8ToString( );
      addTerm ( fieldChars, fieldValue, fieldBuilder, termBuilder );
    }
  }
    
  addDistributedTerms( rb, fieldBuilder, termBuilder, searchFields );
    
  fieldMap = fieldBuilder.build( );
  termMap = termBuilder.build( );
}
 

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
  if (merging) {
    AssertingCodec.assertThread("DocValuesProducer", creationThread);
  }
  assert field.getDocValuesType() == DocValuesType.SORTED_SET;
  SortedSetDocValues values = in.getSortedSet(field);
  assert values != null;
  return new AssertingLeafReader.AssertingSortedSetDocValues(values, maxDoc);
}
 

private static LongBitSet findFieldOrdinalsMatchingQuery(Query q, String field, SolrIndexSearcher searcher, SortedSetDocValues docValues) throws IOException {
  final LongBitSet fromOrdBitSet = new LongBitSet(docValues.getValueCount());
  final Collector fromCollector = new MultiValueTermOrdinalCollector(field, docValues, fromOrdBitSet);

  searcher.search(q, fromCollector);

  return fromOrdBitSet;
}
 

DiskDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec,
    String metaExtension) throws IOException {
  String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
  // read in the entries from the metadata file.
  IndexInput in = state.directory.openInput(metaName, state.context);
  boolean success = false;
  try {
    CodecUtil.checkHeader(in, metaCodec, DiskDocValuesFormat.VERSION_START, DiskDocValuesFormat.VERSION_START);
    numerics = new ConcurrentHashMap<Integer, NumericEntry>();
    ords = new ConcurrentHashMap<Integer, NumericEntry>();
    ordIndexes = new ConcurrentHashMap<Integer, NumericEntry>();
    binaries = new ConcurrentHashMap<Integer, BinaryEntry>();
    _binaryDocValuesCache = new ConcurrentHashMap<Integer, BinaryDocValues>();
    _numericDocValuesCache = new ConcurrentHashMap<Integer, NumericDocValues>();
    _sortedDocValuesCache = new ConcurrentHashMap<Integer, SortedDocValues>();
    _sortedSetDocValuesCache = new ConcurrentHashMap<Integer, SortedSetDocValues>();
    readFields(in, state.fieldInfos);
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }

  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
  data = state.directory.openInput(dataName, state.context);
  CodecUtil.checkHeader(data, dataCodec, DiskDocValuesFormat.VERSION_START, DiskDocValuesFormat.VERSION_START);
}
 

private void assertEquals(int maxDoc, SortedSetDocValues expected, SortedSetDocValues actual) throws Exception {
  // can be null for the segment if no docs actually had any SortedDocValues
  // in this case FC.getDocTermsOrds returns EMPTY
  if (actual == null) {
    assertEquals(expected.getValueCount(), 0);
    return;
  }
  assertEquals(expected.getValueCount(), actual.getValueCount());
  while (true) {
    int docID = expected.nextDoc();
    assertEquals(docID, actual.nextDoc());
    if (docID == NO_MORE_DOCS) {
      break;
    }
    long expectedOrd;
    while ((expectedOrd = expected.nextOrd()) != NO_MORE_ORDS) {
      assertEquals(expectedOrd, actual.nextOrd());
    }
    assertEquals(NO_MORE_ORDS, actual.nextOrd());
  }
  
  // compare ord dictionary
  for (long i = 0; i < expected.getValueCount(); i++) {
    final BytesRef expectedBytes = BytesRef.deepCopyOf(expected.lookupOrd(i));
    final BytesRef actualBytes = actual.lookupOrd(i);
    assertEquals(expectedBytes, actualBytes);
  }
  
  // compare termsenum
  assertEquals(expected.getValueCount(), expected.termsEnum(), actual.termsEnum());
}
 

@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
  if (insaneField.equals(field)) {
    return null;
  } else {
    return in.getSortedSetDocValues(field);
  }
}
 

@Test
public void testDocValues() throws IOException {

  final DocValuesType expectedNumericDvType = Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) ?
    DocValuesType.SORTED_NUMERIC : DocValuesType.SORTED_SET;
  
  assertU(adoc("id", "1", "floatdv", "4.5", "intdv", "-1", "intdv", "3",
      "stringdv", "value1", "stringdv", "value2",
      "booldv", "false", "booldv", "true"));
  assertU(commit());
  try (SolrCore core = h.getCoreInc()) {
    final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
    final SolrIndexSearcher searcher = searcherRef.get();
    try {
      final LeafReader reader = searcher.getSlowAtomicReader();
      assertEquals(1, reader.numDocs());
      final FieldInfos infos = reader.getFieldInfos();
      assertEquals(DocValuesType.SORTED_SET, infos.fieldInfo("stringdv").getDocValuesType());
      assertEquals(DocValuesType.SORTED_SET, infos.fieldInfo("booldv").getDocValuesType());
      assertEquals(expectedNumericDvType, infos.fieldInfo("floatdv").getDocValuesType());
      assertEquals(expectedNumericDvType, infos.fieldInfo("intdv").getDocValuesType());

      SortedSetDocValues dv = reader.getSortedSetDocValues("stringdv");
      assertEquals(0, dv.nextDoc());
      assertEquals(0, dv.nextOrd());
      assertEquals(1, dv.nextOrd());
      assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());

      dv = reader.getSortedSetDocValues("booldv");
      assertEquals(0, dv.nextDoc());
      assertEquals(0, dv.nextOrd());
      assertEquals(1, dv.nextOrd());
      assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());


    } finally {
      searcherRef.decref();
    }
  }
}
 

@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
  SortedSetDocValues values = in.getSortedSetDocValues(field);
  if (values != null) {
    return values;
  }
  Type v = getType(field);
  if (v != null) {
    switch (v) {
      case SORTED_SET_INTEGER:
      case SORTED_SET_FLOAT: 
        return FieldCache.DEFAULT.getDocTermOrds(in, field, FieldCache.INT32_TERM_PREFIX);
      case SORTED_SET_LONG:
      case SORTED_SET_DOUBLE:
        return FieldCache.DEFAULT.getDocTermOrds(in, field, FieldCache.INT64_TERM_PREFIX);
      case SORTED_SET_BINARY:
        return FieldCache.DEFAULT.getDocTermOrds(in, field, null);
      case BINARY:
      case LEGACY_DOUBLE:
      case LEGACY_FLOAT:
      case LEGACY_INTEGER:
      case LEGACY_LONG:
      case DOUBLE_POINT:
      case FLOAT_POINT:
      case INTEGER_POINT:
      case LONG_POINT:
      case SORTED:
        break;
    }
  }
  return null;
}
 

/** Returns a SortedSetDocValues view of this instance */
public SortedSetDocValues iterator(LeafReader reader) throws IOException {
  if (isEmpty()) {
    return DocValues.emptySortedSet();
  } else {
    return new Iterator(reader);
  }
}
 

/** "typical" multi-valued faceting: not too many unique values, no prefixing. maps to global ordinals as a separate step */
static void accumMultiSeg(int counts[], SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  // First count in seg-ord space:
  final int segCounts[];
  if (map == null) {
    segCounts = counts;
  } else {
    segCounts = new int[1+(int)si.getValueCount()];
  }
  
  int doc;
  while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    if (si.advanceExact(doc)) {
      int term = (int) si.nextOrd();
      do {
        segCounts[1+term]++;
      } while ((term = (int)si.nextOrd()) >= 0);
    } else {
      counts[0]++; // missing
    }
  }
  
  // migrate to global ords (if necessary)
  if (map != null) {
    migrateGlobal(counts, segCounts, subIndex, map);
  }
}
 

@Override
public void collect(int doc) throws IOException {
  values.clear();
  if (docValues.advanceExact(doc)) {
    int term;
    while ((term = (int)docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      values.add(docValues.lookupOrd(term).utf8ToString());
    }
  }
}
 

@Override
public void collect(int doc) throws IOException {
  count = 0;
  if (docValues.advanceExact(doc)) {
    int term;
    while ((term = (int)docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      if (count == values.length) {
        resizeValues();
      }
      values[count++] = LegacyNumericUtils.prefixCodedToInt(docValues.lookupOrd(term));
    }
  }
}
 

@Override
public void collect(int doc) throws IOException {
  count = 0;
  if (docValues.advanceExact(doc)) {
    int term;
    while ((term = (int)docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      if (count == values.length) {
        resizeValues();
      }
      values[count++] = NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(docValues.lookupOrd(term)));
    }
  }
}
 

/** accumulates per-segment multi-valued facet counts */
static void accumMulti(int counts[], int startTermIndex, SortedSetDocValues si, DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
  if (startTermIndex == -1 && (map == null || si.getValueCount() < disi.cost()*10)) {
    // no prefixing, not too many unique values wrt matching docs (lucene/facets heuristic): 
    //   collect separately per-segment, then map to global ords
    accumMultiSeg(counts, si, disi, subIndex, map);
  } else {
    // otherwise: do collect+map on the fly
    accumMultiGeneric(counts, startTermIndex, si, disi, subIndex, map);
  }
}
 

@Override
public void collect(int doc) throws IOException {
  count = 0;
  if (docValues.advanceExact(doc)) {
    int term;
    while ((term = (int)docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      if (count == values.length) {
        resizeValues();
      }
      values[count++] = term == trueOrd;
    }
  }
}
 

@Override
public void collect(int doc) throws IOException {
  count = 0;
  if (docValues.advanceExact(doc)) {
    int term;
    while ((term = (int)docValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      if (count == values.length) {
        resizeValues();
      }
      values[count++] = NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(docValues.lookupOrd(term)));
    }
  }
}