Java Code Examples for org.apache.lucene.index.IndexOptions#NONE

/**
 * Create field with Reader value.
 * @param name field name
 * @param reader reader value
 * @param type field type
 * @throws IllegalArgumentException if either the name or type
 *         is null, or if the field's type is stored(), or
 *         if tokenized() is false.
 * @throws NullPointerException if the reader is null
 */
public Field(String name, Reader reader, IndexableFieldType type) {
  if (name == null) {
    throw new IllegalArgumentException("name must not be null");
  }
  if (type == null) {
    throw new IllegalArgumentException("type must not be null");
  }
  if (reader == null) {
    throw new NullPointerException("reader must not be null");
  }
  if (type.stored()) {
    throw new IllegalArgumentException("fields with a Reader value cannot be stored");
  }
  if (type.indexOptions() != IndexOptions.NONE && !type.tokenized()) {
    throw new IllegalArgumentException("non-tokenized fields must use String values");
  }
  
  this.name = name;
  this.fieldsData = reader;
  this.type = type;
}
 

@Override
public int doLogic() throws Exception {
  List<IndexableField> fields = doc.getFields();
  Analyzer analyzer = getRunData().getAnalyzer();
  int tokenCount = 0;
  for(final IndexableField field : fields) {
    if (field.fieldType().indexOptions() == IndexOptions.NONE ||
        field.fieldType().tokenized() == false) {
      continue;
    }
    
    final TokenStream stream = field.tokenStream(analyzer, null);
    // reset the TokenStream to the first token
    stream.reset();

    TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
    while(stream.incrementToken()) {
      termAtt.getBytesRef();
      tokenCount++;
    }
    stream.end();
    stream.close();
  }
  totalTokenCount += tokenCount;
  return tokenCount;
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) {
        return;
    }

    Boolean value = context.parseExternalValue(Boolean.class);
    if (value == null) {
        XContentParser.Token token = context.parser().currentToken();
        if (token == XContentParser.Token.VALUE_NULL) {
            if (fieldType().nullValue() != null) {
                value = fieldType().nullValue();
            }
        } else {
            value = context.parser().booleanValue();
        }
    }

    if (value == null) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), value ? "T" : "F", fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value ? 1 : 0));
    }
}
 

protected void setupFieldType(BuilderContext context) {
    fieldType.setNames(buildNames(context));
    if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) {
        fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
        fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
    }
    if (fieldDataSettings != null) {
        Settings settings = Settings.builder().put(fieldType.fieldDataType().getSettings()).put(fieldDataSettings).build();
        fieldType.setFieldDataType(new FieldDataType(fieldType.fieldDataType().getType(), settings));
    }
    boolean defaultDocValues = false; // pre 2.0
    if (context.indexCreatedVersion().onOrAfter(Version.V_2_0_0_beta1)) {
        defaultDocValues = fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE;
    }
    // backcompat for "fielddata: format: docvalues" for now...
    boolean fieldDataDocValues = fieldType.fieldDataType() != null
        && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldType.fieldDataType().getFormat(context.indexSettings()));
    if (fieldDataDocValues && docValuesSet && fieldType.hasDocValues() == false) {
        // this forces the doc_values setting to be written, so fielddata does not mask the original setting
        defaultDocValues = true;
    }
    defaultFieldType.setHasDocValues(defaultDocValues);
    if (docValuesSet == false) {
        fieldType.setHasDocValues(defaultDocValues || fieldDataDocValues);
    }
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().isEnabled() == false) {
        return;
    }
    for (ParseContext.Document document : context.docs()) {
        final List<String> paths = new ArrayList<>();
        for (IndexableField field : document.getFields()) {
            paths.add(field.name());
        }
        for (String path : paths) {
            for (String fieldName : extractFieldNames(path)) {
                if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
                    document.add(new Field(fieldType().names().indexName(), fieldName, fieldType()));
                }
            }
        }
    }
}
 

public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 

/**
 * Create field with String value.
 * @param name field name
 * @param value string value
 * @param type field type
 * @throws IllegalArgumentException if either the name, value or type
 *         is null, or if the field's type is neither indexed() nor stored(), 
 *         or if indexed() is false but storeTermVectors() is true.
 */
public Field(String name, CharSequence value, IndexableFieldType type) {
  if (name == null) {
    throw new IllegalArgumentException("name must not be null");
  }
  if (value == null) {
    throw new IllegalArgumentException("value must not be null");
  }
  if (type == null) {
    throw new IllegalArgumentException("type must not be null");
  }
  if (!type.stored() && type.indexOptions() == IndexOptions.NONE) {
    throw new IllegalArgumentException("it doesn't make sense to have a field that "
      + "is neither indexed nor stored");
  }
  this.name = name;
  this.fieldsData = value;
  this.type = type;
}
 

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    XContentParser parser = context.parser();
    if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
        // we are in the parse Phase
        String id = parser.text();
        if (context.id() != null && !context.id().equals(id)) {
            throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
        }
        context.id(id);
    } // else we are in the pre/post parse phase

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        fields.add(new Field(fieldType().names().indexName(), context.id(), fieldType()));
    }
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(fieldType().names().indexName(), new BytesRef(context.id())));
    }
}
 

@Override
public IndexOptions indexOptions() {
  if (!indexed()) {
    return IndexOptions.NONE;
  }
  
  IndexOptions options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  if (omitTermFreqAndPositions()) {
    options = IndexOptions.DOCS;
  } else if (omitPositions()) {
    options = IndexOptions.DOCS_AND_FREQS;
  } else if (storeOffsetsWithPositions()) {
    options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  }

  return options;
}
 

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setDoubleValue(number);
    }
    return null;
}
 

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
        BytesRef id = Uid.encodeId(context.sourceToParse().id());
        fields.add(new Field(NAME, id, fieldType));
    }
}
 

@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) {
    if (fieldType().indexOptions() != IndexOptions.NONE) {
        return getCachedStream().setIntValue(number);
    }
    return null;
}
 

/**
 * Create a new instance configured with the provided FieldType options. See {@link #DEFAULT_FIELDTYPE}.
 * a field type is used to articulate the desired options (namely pointValues, docValues, stored).  Legacy numerics
 * is configurable this way too.
 */
public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  this.fieldNameX = fieldNamePrefix+SUFFIX_X;
  this.fieldNameY = fieldNamePrefix+SUFFIX_Y;

  int numPairs = 0;
  if ((this.hasStored = fieldType.stored())) {
    numPairs++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numPairs++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numPairs++;
  }
  if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {
    if (hasPointVals) {
      throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive");
    }
    final LegacyFieldType legacyType = (LegacyFieldType) fieldType;
    if (legacyType.numericType() != LegacyNumericType.DOUBLE) {
      throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType());
    }
    numPairs++;
    legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED);
    legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep());
    legacyNumericFieldType.freeze();
  } else {
    legacyNumericFieldType = null;
  }
  this.fieldsLen = numPairs * 2;
}
 

@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
    if (indexOptions() == IndexOptions.NONE) {
        return new ConstantScoreQuery(new PrefixQuery(new Term(UidFieldMapper.NAME, Uid.typePrefixAsBytes(BytesRefs.toBytesRef(value)))));
    }
    return new ConstantScoreQuery(new TermQuery(createTerm(value)));
}
 

@Override
public StringFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    // if the field is not analyzed, then by default, we should omit norms and have docs only
    // index options, as probably what the user really wants
    // if they are set explicitly, we will use those values
    // we also change the values on the default field type so that toXContent emits what
    // differs from the defaults
    if (fieldType.indexOptions() != IndexOptions.NONE && !fieldType.tokenized()) {
        defaultFieldType.setOmitNorms(true);
        defaultFieldType.setIndexOptions(IndexOptions.DOCS);
        if (!omitNormsSet && fieldType.boost() == 1.0f) {
            fieldType.setOmitNorms(true);
        }
        if (!indexOptionsSet) {
            fieldType.setIndexOptions(IndexOptions.DOCS);
        }
    }
    setupFieldType(context);
    StringFieldMapper fieldMapper = new StringFieldMapper(
            name, fieldType, defaultFieldType, positionIncrementGap, ignoreAbove,
            context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
    return fieldMapper.includeInAll(includeInAll);
}
 

@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
    if (indexOptions() != IndexOptions.NONE || context == null) {
        return super.termQuery(value, context);
    }
    final BytesRef[] uids = Uid.createUidsForTypesAndId(context.queryTypes(), value);
    return new TermsQuery(UidFieldMapper.NAME, uids);
}
 

protected void parse(ParseContext context, GeoPoint point) throws IOException {

        if (ignoreMalformed.value() == false) {
            if (point.lat() > 90.0 || point.lat() < -90.0) {
                throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name());
            }
            if (point.lon() > 180.0 || point.lon() < -180) {
                throw new IllegalArgumentException("illegal longitude value [" + point.lon() + "] for " + name());
            }
        } else {
            GeoUtils.normalizePoint(point);
        }
        if (fieldType().indexOptions() != IndexOptions.NONE) {
            context.doc().add(new LatLonPoint(fieldType().name(), point.lat(), point.lon()));
        }
        if (fieldType().stored()) {
            context.doc().add(new StoredField(fieldType().name(), point.toString()));
        }
        if (fieldType.hasDocValues()) {
            context.doc().add(new LatLonDocValuesField(fieldType().name(), point.lat(), point.lon()));
        } else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) {
            List<IndexableField> fields = new ArrayList<>(1);
            createFieldNamesField(context, fields);
            for (IndexableField field : fields) {
                context.doc().add(field);
            }
        }
        // if the mapping contains multifields then use the geohash string
        if (multiFields.iterator().hasNext()) {
            multiFields.parse(this, context.createExternalValueContext(point.geohash()));
        }
    }
 

@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
    String dateAsString = null;
    float boost = fieldType().boost();
    if (context.externalValueSet()) {
        Object externalValue = context.externalValue();
        dateAsString = (String) externalValue;
        if (dateAsString == null) {
            dateAsString = fieldType().nullValueAsString();
        }
    } else {
        XContentParser parser = context.parser();
        XContentParser.Token token = parser.currentToken();
        if (token == XContentParser.Token.VALUE_NULL) {
            dateAsString = fieldType().nullValueAsString();
        } else if (token == XContentParser.Token.VALUE_NUMBER) {
            dateAsString = parser.text();
        } else if (token == XContentParser.Token.START_OBJECT) {
            String currentFieldName = null;
            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                if (token == XContentParser.Token.FIELD_NAME) {
                    currentFieldName = parser.currentName();
                } else {
                    if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
                        if (token == XContentParser.Token.VALUE_NULL) {
                            dateAsString = fieldType().nullValueAsString();
                        } else {
                            dateAsString = parser.text();
                        }
                    } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
                        boost = parser.floatValue();
                    } else {
                        throw new IllegalArgumentException("unknown property [" + currentFieldName + "]");
                    }
                }
            }
        } else {
            dateAsString = parser.text();
        }
    }

    Long value = null;
    if (dateAsString != null) {
        if (context.includeInAll(includeInAll, this)) {
            context.allEntries().addText(fieldType().names().fullName(), dateAsString, boost);
        }
        value = fieldType().parseStringValue(dateAsString);
    }

    if (value != null) {
        if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
            CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
            field.setBoost(boost);
            fields.add(field);
        }
        if (fieldType().hasDocValues()) {
            addDocValue(context, fields, value);
        }
    }
}
 

@Override
public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException {
  if (parser == null) {
    throw new NullPointerException();
  }
  final NumericDocValues valuesIn = reader.getNumericDocValues(field);
  if (valuesIn != null) {
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptyNumeric();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    }
    
    if (parser instanceof PointParser) {
      // points case
      // no points in this segment
      if (info.getPointDimensionCount() == 0) {
        return DocValues.emptyNumeric();
      }
      if (info.getPointDimensionCount() != 1) {
        throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount());
      }
      PointValues values = reader.getPointValues(field);
      // no actual points for this field (e.g. all points deleted)
      if (values == null || values.size() == 0) {
        return DocValues.emptyNumeric();
      }
      // not single-valued
      if (values.size() != values.getDocCount()) {
        throw new IllegalStateException("Type mismatch: " + field + " was indexed with multiple values, numValues=" + values.size() + ",numDocs=" + values.getDocCount());
      }
    } else {
      // postings case 
      // not indexed
      if (info.getIndexOptions() == IndexOptions.NONE) {
        return DocValues.emptyNumeric();
      }
    }

    return ((LongsFromArray) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser))).iterator();
  }
}
 

/**
 * Checks for any conflicts between this field type and other.
 * If strict is true, all properties must be equal.
 * Otherwise, only properties which must never change in an index are checked.
 */
public void checkCompatibility(MappedFieldType other, List<String> conflicts, boolean strict) {
    checkTypeName(other);

    boolean indexed = indexOptions() != IndexOptions.NONE;
    boolean mergeWithIndexed = other.indexOptions() != IndexOptions.NONE;
    // TODO: should be validating if index options go "up" (but "down" is ok)
    if (indexed != mergeWithIndexed) {
        conflicts.add("mapper [" + name() + "] has different [index] values");
    }
    if (stored() != other.stored()) {
        conflicts.add("mapper [" + name() + "] has different [store] values");
    }
    if (hasDocValues() != other.hasDocValues()) {
        conflicts.add("mapper [" + name() + "] has different [doc_values] values");
    }
    if (omitNorms() && !other.omitNorms()) {
        conflicts.add("mapper [" + name() + "] has different [norms] values, cannot change from disable to enabled");
    }
    if (storeTermVectors() != other.storeTermVectors()) {
        conflicts.add("mapper [" + name() + "] has different [store_term_vector] values");
    }
    if (storeTermVectorOffsets() != other.storeTermVectorOffsets()) {
        conflicts.add("mapper [" + name() + "] has different [store_term_vector_offsets] values");
    }
    if (storeTermVectorPositions() != other.storeTermVectorPositions()) {
        conflicts.add("mapper [" + name() + "] has different [store_term_vector_positions] values");
    }
    if (storeTermVectorPayloads() != other.storeTermVectorPayloads()) {
        conflicts.add("mapper [" + name() + "] has different [store_term_vector_payloads] values");
    }

    // null and "default"-named index analyzers both mean the default is used
    if (indexAnalyzer() == null || "default".equals(indexAnalyzer().name())) {
        if (other.indexAnalyzer() != null && "default".equals(other.indexAnalyzer().name()) == false) {
            conflicts.add("mapper [" + name() + "] has different [analyzer]");
        }
    } else if (other.indexAnalyzer() == null || "default".equals(other.indexAnalyzer().name())) {
        conflicts.add("mapper [" + name() + "] has different [analyzer]");
    } else if (indexAnalyzer().name().equals(other.indexAnalyzer().name()) == false) {
        conflicts.add("mapper [" + name() + "] has different [analyzer]");
    }

    if (strict) {
        if (omitNorms() != other.omitNorms()) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [omit_norms] across all types.");
        }
        if (boost() != other.boost()) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [boost] across all types.");
        }
        if (Objects.equals(searchAnalyzer(), other.searchAnalyzer()) == false) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_analyzer] across all types.");
        }
        if (Objects.equals(searchQuoteAnalyzer(), other.searchQuoteAnalyzer()) == false) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_quote_analyzer] across all types.");
        }
        if (Objects.equals(nullValue(), other.nullValue()) == false) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [null_value] across all types.");
        }
        if (eagerGlobalOrdinals() != other.eagerGlobalOrdinals()) {
            conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [eager_global_ordinals] across all types.");
        }
    }
}