org.apache.lucene.index.DocValuesType Java Examples

The following examples show how to use org.apache.lucene.index.DocValuesType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FacetFieldProcessorByHashDV.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) {
  super(fcontext, freq, sf);
  if (freq.mincount == 0) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support mincount=0");
  }
  if (freq.prefix != null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support prefix"); // yet, but it could
  }
  FieldInfo fieldInfo = fcontext.searcher.getFieldInfos().fieldInfo(sf.getName());
  if (fieldInfo != null &&
      fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED_NUMERIC) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" only support single valued number/string with docValues");
  }
}
 
Example #2
Source File: TestMemoryIndex.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testToStringDebug() {
  MemoryIndex mi = new MemoryIndex(true, true);
  Analyzer analyzer = new MockPayloadAnalyzer();

  mi.addField("analyzedField", "aa bb aa", analyzer);

  FieldType type = new FieldType();
  type.setDimensions(1, 4);
  type.setDocValuesType(DocValuesType.BINARY);
  type.freeze();
  mi.addField(new BinaryPoint("pointAndDvField", "term".getBytes(StandardCharsets.UTF_8), type), analyzer);

  assertEquals("analyzedField:\n" +
      "\t'[61 61]':2: [(0, 0, 2, [70 6f 73 3a 20 30]), (1, 6, 8, [70 6f 73 3a 20 32])]\n" +
      "\t'[62 62]':1: [(1, 3, 5, [70 6f 73 3a 20 31])]\n" +
      "\tterms=2, positions=3\n" +
      "pointAndDvField:\n" +
      "\tterms=0, positions=0\n" +
      "\n" +
      "fields=2, terms=2, positions=3", mi.toStringDebug());
}
 
Example #3
Source File: TestMemoryIndex.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIndexingPointsAndDocValues() throws Exception {
  FieldType type = new FieldType();
  type.setDimensions(1, 4);
  type.setDocValuesType(DocValuesType.BINARY);
  type.freeze();
  Document doc = new Document();
  byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8);
  doc.add(new BinaryPoint("field", packedPoint, type));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  assertEquals(1, leafReader.getPointValues("field").size());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMinPackedValue());
  assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMaxPackedValue());

  BinaryDocValues dvs = leafReader.getBinaryDocValues("field");
  assertEquals(0, dvs.nextDoc());
  assertEquals("term", dvs.binaryValue().utf8ToString());
}
 
Example #4
Source File: PointVectorStrategy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new instance configured with the provided FieldType options. See {@link #DEFAULT_FIELDTYPE}.
 * a field type is used to articulate the desired options (namely pointValues, docValues, stored).  Legacy numerics
 * is configurable this way too.
 */
public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  this.fieldNameX = fieldNamePrefix+SUFFIX_X;
  this.fieldNameY = fieldNamePrefix+SUFFIX_Y;

  int numPairs = 0;
  if ((this.hasStored = fieldType.stored())) {
    numPairs++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numPairs++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numPairs++;
  }
  this.fieldsLen = numPairs * 2;
}
 
Example #5
Source File: ParentFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public ParentFieldMapper build(BuilderContext context) {
    if (parentType == null) {
        throw new MapperParsingException("[_parent] field mapping must contain the [type] option");
    }
    parentJoinFieldType.setNames(new MappedFieldType.Names(joinField(documentType)));
    parentJoinFieldType.setFieldDataType(null);
    childJoinFieldType.setNames(new MappedFieldType.Names(joinField(parentType)));
    if (context.indexCreatedVersion().before(Version.V_2_0_0_beta1)) {
        childJoinFieldType.setHasDocValues(false);
        childJoinFieldType.setDocValuesType(DocValuesType.NONE);
        parentJoinFieldType.setHasDocValues(false);
        parentJoinFieldType.setDocValuesType(DocValuesType.NONE);
    }
    return new ParentFieldMapper(fieldType, parentJoinFieldType, childJoinFieldType, parentType, context.indexSettings());
}
 
Example #6
Source File: VersionFieldUpgrader.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
static CodecReader wrap(CodecReader reader) throws IOException {
    final FieldInfos fieldInfos = reader.getFieldInfos();
    final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME);
    if (versionInfo != null && versionInfo.getDocValuesType() != DocValuesType.NONE) {
        // the reader is a recent one, it has versions and they are stored
        // in a numeric doc values field
        return reader;
    }
    // The segment is an old one, look at the _uid field
    final Terms terms = reader.terms(UidFieldMapper.NAME);
    if (terms == null || !terms.hasPayloads()) {
        // The segment doesn't have an _uid field or doesn't have payloads
        // don't try to do anything clever. If any other segment has versions
        // all versions of this segment will be initialized to 0
        return reader;
    }
    // convert _uid payloads -> _version docvalues
    return new VersionFieldUpgrader(reader);
}
 
Example #7
Source File: DocValuesAdapter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the doc values for the specified field in the specified document.
 * Empty Optional instance is returned if no doc values is available for the field.
 *
 * @param docid - document id
 * @param field - field name
 * @return doc values, if exists, or empty
 * @throws IOException - if there is a low level IO error.
 */
Optional<DocValues> getDocValues(int docid, String field) throws IOException {
  DocValuesType dvType = IndexUtils.getFieldInfo(reader, field).getDocValuesType();

  switch (dvType) {
    case BINARY:
      return createBinaryDocValues(docid, field, DocValuesType.BINARY);
    case NUMERIC:
      return createNumericDocValues(docid, field, DocValuesType.NUMERIC);
    case SORTED_NUMERIC:
      return createSortedNumericDocValues(docid, field, DocValuesType.SORTED_NUMERIC);
    case SORTED:
      return createSortedDocValues(docid, field, DocValuesType.SORTED);
    case SORTED_SET:
      return createSortedSetDocValues(docid, field, DocValuesType.SORTED_SET);
    default:
      return Optional.empty();
  }
}
 
Example #8
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
  if (valuesIn == null) {
    valuesIn = reader.getSortedDocValues(field);
  }

  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  }

  final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info == null) {
    return DocValues.emptyBinary();
  } else if (info.getDocValuesType() != DocValuesType.NONE) {
    throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
  } else if (info.getIndexOptions() == IndexOptions.NONE) {
    return DocValues.emptyBinary();
  }

  BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
  return impl.iterator();
}
 
Example #9
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 
Example #10
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException {
  final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  if (fieldInfo == null) {
    // field does not exist or has no value
    return new Bits.MatchNoBits(reader.maxDoc());
  } 
  
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
    // doc values case
  } else if (parser instanceof PointParser) {
    // points case
  } else {
    // postings case
    if (fieldInfo.getIndexOptions() == IndexOptions.NONE) {
      return new Bits.MatchNoBits(reader.maxDoc());
    }
  }
  BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser));
  return bitsEntry.bits;
}
 
Example #11
Source File: DocValuesAdapter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private Optional<DocValues> createSortedSetDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedSetDocValues ssvalues = IndexUtils.getSortedSetDocvalues(reader, field);

  if (ssvalues.advanceExact(docid)) {
    List<BytesRef> values = new ArrayList<>();

    long ord;
    while ((ord = ssvalues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
      values.add(BytesRef.deepCopyOf(ssvalues.lookupOrd(ord)));
    }

    DocValues dv = DocValues.of(
        dvType,
        values,
        Collections.emptyList()
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 
Example #12
Source File: Lucene60FieldInfosFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
  switch(b) {
  case 0:
    return DocValuesType.NONE;
  case 1:
    return DocValuesType.NUMERIC;
  case 2:
    return DocValuesType.BINARY;
  case 3:
    return DocValuesType.SORTED;
  case 4:
    return DocValuesType.SORTED_SET;
  case 5:
    return DocValuesType.SORTED_NUMERIC;
  default:
    throw new CorruptIndexException("invalid docvalues byte: " + b, input);
  }
}
 
Example #13
Source File: Lucene60FieldInfosFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static byte docValuesByte(DocValuesType type) {
  switch(type) {
  case NONE:
    return 0;
  case NUMERIC:
    return 1;
  case BINARY:
    return 2;
  case SORTED:
    return 3;
  case SORTED_SET:
    return 4;
  case SORTED_NUMERIC:
    return 5;
  default:
    // BUG
    throw new AssertionError("unhandled DocValuesType: " + type);
  }
}
 
Example #14
Source File: TestBlockWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static FieldInfo getMockFieldInfo(String fieldName, int number) {
  return new FieldInfo(fieldName,
      number,
      false,
      false,
      true,
      IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
      DocValuesType.NONE,
      -1,
      Collections.emptyMap(),
      0,
      0,
      0,
      true
  );
}
 
Example #15
Source File: TestSTBlockReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static FieldInfo mockFieldInfo(String fieldName, int number) {
  return new FieldInfo(fieldName,
      number,
      false,
      false,
      true,
      IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
      DocValuesType.NONE,
      -1,
      Collections.emptyMap(),
      0,
      0,
      0,
      false
  );
}
 
Example #16
Source File: Lucene50FieldInfosFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static byte docValuesByte(DocValuesType type) {
  switch(type) {
  case NONE:
    return 0;
  case NUMERIC:
    return 1;
  case BINARY:
    return 2;
  case SORTED:
    return 3;
  case SORTED_SET:
    return 4;
  case SORTED_NUMERIC:
    return 5;
  default:
    // BUG
    throw new AssertionError("unhandled DocValuesType: " + type);
  }
}
 
Example #17
Source File: Lucene50FieldInfosFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
  switch(b) {
  case 0:
    return DocValuesType.NONE;
  case 1:
    return DocValuesType.NUMERIC;
  case 2:
    return DocValuesType.BINARY;
  case 3:
    return DocValuesType.SORTED;
  case 4:
    return DocValuesType.SORTED_SET;
  case 5:
    return DocValuesType.SORTED_NUMERIC;
  default:
    throw new CorruptIndexException("invalid docvalues byte: " + b, input);
  }
}
 
Example #18
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
ReaderWrapper(LeafReader leafReader, String field) {
  super(leafReader);

  // TODO can we just do "field" and not bother with the other fields?
  List<FieldInfo> newInfos = new ArrayList<>(in.getFieldInfos().size());
  for (FieldInfo fieldInfo : in.getFieldInfos()) {
    if (fieldInfo.name.equals(field)) {
      FieldInfo f = new FieldInfo(fieldInfo.name,
          fieldInfo.number,
          fieldInfo.hasVectors(),
          fieldInfo.hasNorms(),
          fieldInfo.hasPayloads(),
          fieldInfo.getIndexOptions(),
          DocValuesType.NONE,
          fieldInfo.getDocValuesGen(),
          fieldInfo.attributes(),
          fieldInfo.getPointDimensionCount(),
          fieldInfo.getPointIndexDimensionCount(),
          fieldInfo.getPointNumBytes(),
          fieldInfo.isSoftDeletesField());
      newInfos.add(f);
    } else {
      newInfos.add(fieldInfo);
    }
  }
  FieldInfos infos = new FieldInfos(newInfos.toArray(new FieldInfo[newInfos.size()]));
  this.fieldInfos = infos;
}
 
Example #19
Source File: PointVectorStrategy.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Create a new instance configured with the provided FieldType options. See {@link #DEFAULT_FIELDTYPE}.
 * a field type is used to articulate the desired options (namely pointValues, docValues, stored).  Legacy numerics
 * is configurable this way too.
 */
public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  this.fieldNameX = fieldNamePrefix+SUFFIX_X;
  this.fieldNameY = fieldNamePrefix+SUFFIX_Y;

  int numPairs = 0;
  if ((this.hasStored = fieldType.stored())) {
    numPairs++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numPairs++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numPairs++;
  }
  if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {
    if (hasPointVals) {
      throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive");
    }
    final LegacyFieldType legacyType = (LegacyFieldType) fieldType;
    if (legacyType.numericType() != LegacyNumericType.DOUBLE) {
      throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType());
    }
    numPairs++;
    legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED);
    legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep());
    legacyNumericFieldType.freeze();
  } else {
    legacyNumericFieldType = null;
  }
  this.fieldsLen = numPairs * 2;
}
 
Example #20
Source File: SolrDocumentFetcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) {
  if (!schemaField.hasDocValues() || !schemaField.stored()) return false;
  if (schemaField.multiValued()) return false;
  DocValuesType docValuesType = fieldInfo.getDocValuesType();
  NumberType numberType = schemaField.getType().getNumberType();
  // can not decode a numeric without knowing its numberType
  if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) {
    return false;
  }
  return true;
}
 
Example #21
Source File: LatLonDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */
static void checkCompatible(FieldInfo fieldInfo) {
  // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) {
    throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + 
                                       " but this type has docValuesType=" + TYPE.docValuesType() + 
                                       ", is the field really a LatLonDocValuesField?");
  }
}
 
Example #22
Source File: FloatPointField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Object toObject(IndexableField f) {
  final Number val = f.numericValue();
  if (val != null) {
    if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.NUMERIC) {
      return Float.intBitsToFloat(val.intValue());
    } else if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) {
      return NumericUtils.sortableIntToFloat(val.intValue());
    } else  {
      return val;
    }
  } else {
    throw new AssertionError("Unexpected state. Field: '" + f + "'");
  }
}
 
Example #23
Source File: BBoxField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected BBoxStrategy newSpatialStrategy(String fieldName) {
  //if it's a dynamic field, we register the sub-fields now.
  FieldType numberType = schema.getFieldTypeByName(numberTypeName);
  FieldType booleanType = schema.getFieldTypeByName(booleanTypeName);
  if (schema.isDynamicField(fieldName)) {
    registerSubFields(schema, fieldName, numberType, booleanType);
  }

  //Solr's FieldType ought to expose Lucene FieldType. Instead as a hack we create a Field with a dummy value.
  final SchemaField solrNumField = new SchemaField("_", numberType);//dummy temp
  org.apache.lucene.document.FieldType luceneType =
      (org.apache.lucene.document.FieldType) solrNumField.createField(0.0).fieldType();
  if ( ! (luceneType instanceof LegacyFieldType)) {
    luceneType = new org.apache.lucene.document.FieldType(luceneType);
  }
  luceneType.setStored(storeSubFields);
  
  //and annoyingly this Field isn't going to have a docValues format because Solr uses a separate Field for that
  if (solrNumField.hasDocValues()) {
    if (luceneType instanceof LegacyFieldType) {
      luceneType = new LegacyFieldType((LegacyFieldType)luceneType);
    } else {
      luceneType = new org.apache.lucene.document.FieldType(luceneType);
    }
    luceneType.setDocValuesType(DocValuesType.NUMERIC);
  }
  return new BBoxStrategy(ctx, fieldName, luceneType);
}
 
Example #24
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public FieldsReader(final SegmentReadState readState) throws IOException {

      // Init each unique format:
      boolean success = false;
      try {
        // Read field name -> format name
        for (FieldInfo fi : readState.fieldInfos) {
          if (fi.getDocValuesType() != DocValuesType.NONE) {
            final String fieldName = fi.name;
            final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY);
            if (formatName != null) {
              // null formatName means the field is in fieldInfos, but has no docvalues!
              final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY);
              if (suffix == null) {
                throw new IllegalStateException("missing attribute: " + PER_FIELD_SUFFIX_KEY + " for field: " + fieldName);
              }
              DocValuesFormat format = DocValuesFormat.forName(formatName);
              String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix));
              if (!formats.containsKey(segmentSuffix)) {
                formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
              }
              fields.put(fieldName, formats.get(segmentSuffix));
            }
          }
        }
        success = true;
      } finally {
        if (!success) {
          IOUtils.closeWhileHandlingException(formats.values());
        }
      }
    }
 
Example #25
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void merge(MergeState mergeState) throws IOException {
  Map<DocValuesConsumer, Collection<String>> consumersToField = new IdentityHashMap<>();

  // Group each consumer by the fields it handles
  for (FieldInfo fi : mergeState.mergeFieldInfos) {
    if (fi.getDocValuesType() == DocValuesType.NONE) {
      continue;
    }
    // merge should ignore current format for the fields being merged
    DocValuesConsumer consumer = getInstance(fi, true);
    Collection<String> fieldsForConsumer = consumersToField.get(consumer);
    if (fieldsForConsumer == null) {
      fieldsForConsumer = new ArrayList<>();
      consumersToField.put(consumer, fieldsForConsumer);
    }
    fieldsForConsumer.add(fi.name);
  }

  // Delegate the merge to the appropriate consumer
  PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
  try {
    for (Map.Entry<DocValuesConsumer, Collection<String>> e : consumersToField.entrySet()) {
      e.getKey().merge(pfMergeState.apply(e.getValue()));
    }
  } finally {
    pfMergeState.reset();
  }
}
 
Example #26
Source File: PerFieldMergeState.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
FilterFieldInfos(FieldInfos src, Collection<String> filterFields) {
  // Copy all the input FieldInfo objects since the field numbering must be kept consistent
  super(toArray(src));

  boolean hasVectors = false;
  boolean hasProx = false;
  boolean hasPayloads = false;
  boolean hasOffsets = false;
  boolean hasFreq = false;
  boolean hasNorms = false;
  boolean hasDocValues = false;
  boolean hasPointValues = false;

  this.filteredNames = new HashSet<>(filterFields);
  this.filtered = new ArrayList<>(filterFields.size());
  for (FieldInfo fi : src) {
    if (this.filteredNames.contains(fi.name)) {
      this.filtered.add(fi);
      hasVectors |= fi.hasVectors();
      hasProx |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
      hasFreq |= fi.getIndexOptions() != IndexOptions.DOCS;
      hasOffsets |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
      hasNorms |= fi.hasNorms();
      hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE;
      hasPayloads |= fi.hasPayloads();
      hasPointValues |= (fi.getPointDimensionCount() != 0);
    }
  }

  this.filteredHasVectors = hasVectors;
  this.filteredHasProx = hasProx;
  this.filteredHasPayloads = hasPayloads;
  this.filteredHasOffsets = hasOffsets;
  this.filteredHasFreq = hasFreq;
  this.filteredHasNorms = hasNorms;
  this.filteredHasDocValues = hasDocValues;
  this.filteredHasPointValues = hasPointValues;
}
 
Example #27
Source File: DoublePointField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Object toObject(IndexableField f) {
  final Number val = f.numericValue();
  if (val != null) {
    if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.NUMERIC) {
      return Double.longBitsToDouble(val.longValue());
    } else if (f.fieldType().stored() == false && f.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) {
      return NumericUtils.sortableLongToDouble(val.longValue());
    } else {
      return val;
    }
  } else {
    throw new AssertionError("Unexpected state. Field: '" + f + "'");
  }
}
 
Example #28
Source File: VersionFieldUpgrader.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
VersionFieldUpgrader(CodecReader in) {
    super(in);

    // Find a free field number
    int fieldNumber = 0;
    for (FieldInfo fi : in.getFieldInfos()) {
        fieldNumber = Math.max(fieldNumber, fi.number + 1);
    }
        
    // TODO: lots of things can wrong here...
    FieldInfo newInfo = new FieldInfo(VersionFieldMapper.NAME,               // field name
                                      fieldNumber,                           // field number
                                      false,                                 // store term vectors
                                      false,                                 // omit norms
                                      false,                                 // store payloads
                                      IndexOptions.NONE,                     // index options
                                      DocValuesType.NUMERIC,                 // docvalues
                                      -1,                                    // docvalues generation
                                      Collections.<String, String>emptyMap() // attributes
                                      );
    newInfo.checkConsistency(); // fail merge immediately if above code is wrong
    
    final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>();
    for (FieldInfo info : in.getFieldInfos()) {
        if (!info.name.equals(VersionFieldMapper.NAME)) {
            fieldInfoList.add(info);
        }
    }
    fieldInfoList.add(newInfo);
    infos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()]));
}
 
Example #29
Source File: DocValuesConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Merges in the fields from the readers in 
 *  <code>mergeState</code>. The default implementation 
 *  calls {@link #mergeNumericField}, {@link #mergeBinaryField},
 *  {@link #mergeSortedField}, {@link #mergeSortedSetField},
 *  or {@link #mergeSortedNumericField} for each field,
 *  depending on its type.
 *  Implementations can override this method 
 *  for more sophisticated merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
  for(DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {
    if (docValuesProducer != null) {
      docValuesProducer.checkIntegrity();
    }
  }

  for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
    DocValuesType type = mergeFieldInfo.getDocValuesType();
    if (type != DocValuesType.NONE) {
      if (type == DocValuesType.NUMERIC) {
        mergeNumericField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.BINARY) {
        mergeBinaryField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED) {
        mergeSortedField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_SET) {
        mergeSortedSetField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_NUMERIC) {
        mergeSortedNumericField(mergeFieldInfo, mergeState);
      } else {
        throw new AssertionError("type=" + type);
      }
    }
  }
}
 
Example #30
Source File: TestFieldType.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testEquals() throws Exception {
  FieldType ft = new FieldType();
  assertEquals(ft, ft);
  assertFalse(ft.equals(null));
  
  FieldType ft2 = new FieldType();
  assertEquals(ft, ft2);
  assertEquals(ft.hashCode(), ft2.hashCode());
  
  FieldType ft3 = new FieldType();
  ft3.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  assertFalse(ft3.equals(ft));
  
  FieldType ft4 = new FieldType();
  ft4.setDocValuesType(DocValuesType.BINARY);
  assertFalse(ft4.equals(ft));
  
  FieldType ft5 = new FieldType();
  ft5.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
  assertFalse(ft5.equals(ft));
  
  FieldType ft6 = new FieldType();
  ft6.setStored(true);
  assertFalse(ft6.equals(ft));
  
  FieldType ft7 = new FieldType();
  ft7.setOmitNorms(true);
  assertFalse(ft7.equals(ft));
  
  FieldType ft10 = new FieldType();
  ft10.setStoreTermVectors(true);
  assertFalse(ft10.equals(ft));
  
  FieldType ft11 = new FieldType();
  ft11.setDimensions(1, 4);
  assertFalse(ft11.equals(ft));
}