Java Code Examples for org.apache.solr.schema.SchemaField#stored()

/**
 * Returns a collection of the names of all stored fields which can be highlighted the index reader knows about.
 */
public Collection<String> getStoredHighlightFieldNames() {
  synchronized (this) {
    if (storedHighlightFieldNames == null) {
      storedHighlightFieldNames = new LinkedList<>();
      for (FieldInfo fieldInfo : searcher.getFieldInfos()) {
        final String fieldName = fieldInfo.name;
        try {
          SchemaField field = searcher.getSchema().getField(fieldName);
          if (field.stored() && ((field.getType() instanceof org.apache.solr.schema.TextField)
              || (field.getType() instanceof org.apache.solr.schema.StrField))) {
            storedHighlightFieldNames.add(fieldName);
          }
        } catch (RuntimeException e) { // getField() throws a SolrException, but it arrives as a RuntimeException
          log.warn("Field [{}] found in index, but not defined in schema.", fieldName);
        }
      }
    }
    return storedHighlightFieldNames;
  }
}
 

private static SolrInputDocument toSolrInputDocument(Document doc, IndexSchema schema) {
  SolrInputDocument out = new SolrInputDocument();
  for( IndexableField f : doc.getFields() ) {
    String fname = f.name();
    SchemaField sf = schema.getFieldOrNull(f.name());
    Object val = null;
    if (sf != null) {
      if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue;
      val = sf.getType().toObject(f);   // object or external string?
    } else {
      val = f.stringValue();
      if (val == null) val = f.numericValue();
      if (val == null) val = f.binaryValue();
      if (val == null) val = f;
    }

    // todo: how to handle targets of copy fields (including polyfield sub-fields)?
    out.addField(fname, val);
  }
  return out;
}
 

IndexSchema addFields(IndexSchema schema) {

    List<SchemaField> fieldsToAdd = new ArrayList<>();

    for (RetrieveField field : fields.values()) {
      allFields.add(field);
      SchemaField schemaField = field.schemaField;
      fieldsToAdd.add(schemaField);
      if (schemaField.multiValued()) {
        multiValuedFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored() == false) {
        dvNotStoredFields.add(field);
      }
      if (schemaField.hasDocValues() == false && schemaField.stored()) {
        storedNotDvFields.add(field);
      }
      if (schemaField.hasDocValues() && schemaField.stored()) {
        storedAndDvFields.add(field);
      }
      if (schemaField.stored() && schemaField.multiValued()) {
        storedMvFields.add(field);
      }
    }
    return schema.addFields(fieldsToAdd, Collections.emptyMap(), false);
  }
 

private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      SchemaField field = schema.getField(fieldName);
      if (field.stored() && field.getType() instanceof StrField ) {
        strFields.add( fieldName );
      }
    }
  }
    
  return strFields;
}
 

private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      try {
        SchemaField field = schema.getField(fieldName);
        if (field.stored() && field.getType() instanceof StrField ) {
          strFields.add( fieldName );
        }
      }
      catch (Throwable e )
      {
          
      }
    }
  }
    
  return strFields;
}
 

/**
 * Returns true if field needs to be skipped else false
 * @param field name of the field
 * @return boolean value
 */
public boolean shouldSkipField(String field) {
  Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
  SchemaField sf = schema.getFieldOrNull(field);

  // Return stored fields or useDocValuesAsStored=true fields,
  // unless an explicit field list is specified
  return  (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
      sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()));
}
 

/**
 * Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the specified
 * schema, after verifying that it is indexed, stored, and single-valued.  
 * If any of these pre-conditions are not met, it throws a SolrException 
 * with a user suitable message indicating the problem.
 */
public static SchemaField getAndCheckVersionField(IndexSchema schema) 
  throws SolrException {
  final String errPrefix = VERSION_FIELD + " field must exist in schema and be searchable (indexed or docValues) and retrievable(stored or docValues) and not multiValued";
  SchemaField sf = schema.getFieldOrNull(VERSION_FIELD);

  if (null == sf) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " does not exist)");
  }
  if ( !sf.indexed() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not searchable");
  }
  if ( !sf.stored() && !sf.hasDocValues()) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " not retrievable");
  }
  if ( sf.multiValued() ) {
    throw new SolrException
      (SolrException.ErrorCode.SERVER_ERROR, 
       errPrefix + " (" + VERSION_FIELD + " is multiValued");
  }
  
  return sf;
}
 

@Override
public void inform(SolrCore core) {

  if (core.getUpdateHandler().getUpdateLog() == null) {
    throw new SolrException(SERVER_ERROR,
        "updateLog must be enabled.");
  }

  if (core.getLatestSchema().getUniqueKeyField() == null) {
    throw new SolrException(SERVER_ERROR,
        "schema must have uniqueKey defined.");
  }

  useFieldCache = true;
  for (String versionField : versionFields) {
    SchemaField userVersionField = core.getLatestSchema().getField(versionField);
    if (userVersionField == null || !userVersionField.stored() || userVersionField.multiValued()) {
      throw new SolrException(SERVER_ERROR,
          "field " + versionField + " must be defined in schema, be stored, and be single valued.");
    }
    if (useFieldCache) {
      try {
        userVersionField.getType().getValueSource(userVersionField, null);
      } catch (Exception e) {
        useFieldCache = false;
        log.warn("Can't use fieldcache/valuesource: {}", e.getMessage());
      }
    }
  }
  
  canCreateTombstoneDocument(core.getLatestSchema());
}
 

private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) {
  if (!schemaField.hasDocValues() || !schemaField.stored()) return false;
  if (schemaField.multiValued()) return false;
  DocValuesType docValuesType = fieldInfo.getDocValuesType();
  NumberType numberType = schemaField.getType().getNumberType();
  // can not decode a numeric without knowing its numberType
  if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) {
    return false;
  }
  return true;
}
 

private static SolrInputDocument toSolrInputDocument(SolrDocument doc, IndexSchema schema) {
  SolrInputDocument out = new SolrInputDocument();
  for( String fname : doc.getFieldNames() ) {
    boolean fieldArrayListCreated = false;
    SchemaField sf = schema.getFieldOrNull(fname);
    if (sf != null) {
      if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue;
    }
    for (Object val: doc.getFieldValues(fname)) {
      if (val instanceof Field) {
        Field f = (Field) val;
        if (sf != null) {
          val = sf.getType().toObject(f);   // object or external string?
        } else {
          val = f.stringValue();
          if (val == null) val = f.numericValue();
          if (val == null) val = f.binaryValue();
          if (val == null) val = f;
        }
      } else if(val instanceof SolrDocument) {
        val = toSolrInputDocument((SolrDocument) val, schema);
        if(!fieldArrayListCreated && doc.getFieldValue(fname) instanceof Collection) {
          // previous value was array so we must return as an array even if was a single value array
          out.setField(fname, Lists.newArrayList(val));
          fieldArrayListCreated = true;
          continue;
        }
      }
      out.addField(fname, val);
    }
  }
  return out;
}
 

@Override
    public IndexableField createField(SchemaField field, Object val /*, float boost*/) {
        if (val == null) return null;
        if (!field.stored()) {
            return null;
        }
        byte[] buf = null;
        int offset = 0, len = 0;
        if (val instanceof byte[]) {
            buf = (byte[]) val;
            len = buf.length;
        } else if (val instanceof ByteBuffer && ((ByteBuffer)val).hasArray()) {
            ByteBuffer byteBuf = (ByteBuffer) val;
            buf = byteBuf.array();
            offset = byteBuf.position();
            len = byteBuf.limit() - byteBuf.position();
        } else {
            String strVal = val.toString();
            //the string has to be a base64 encoded string
            buf = Base64.base64ToByteArray(strVal);
            offset = 0;
            len = buf.length;
        }

        Field f = new org.apache.lucene.document.BinaryDocValuesField(field.getName(), new BytesRef(buf, offset, len));
//        Field f = new org.apache.lucene.document.StoredField(field.getName(), buf, offset, len);
        //f.setBoost(boost);
        return f;
    }
 

@SuppressWarnings({"unchecked"})
SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) {
  this.searcher = searcher;
  this.enableLazyFieldLoading = solrConfig.enableLazyFieldLoading;
  if (cachingEnabled) {
    documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
  } else {
    documentCache = null;
  }

  final Set<String> nonStoredDVsUsedAsStored = new HashSet<>();
  final Set<String> allNonStoredDVs = new HashSet<>();
  final Set<String> nonStoredDVsWithoutCopyTargets = new HashSet<>();
  final Set<String> storedLargeFields = new HashSet<>();
  final Set<String> dvsCanSubstituteStored = new HashSet<>();
  final Set<String> allStoreds = new HashSet<>();

  for (FieldInfo fieldInfo : searcher.getFieldInfos()) { // can find materialized dynamic fields, unlike using the Solr IndexSchema.
    final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldInfo.name);
    if (schemaField == null) {
      continue;
    }
    if (canSubstituteDvForStored(fieldInfo, schemaField)) {
      dvsCanSubstituteStored.add(fieldInfo.name);
    }
    if (schemaField.stored()) {
      allStoreds.add(fieldInfo.name);
    }
    if (!schemaField.stored() && schemaField.hasDocValues()) {
      if (schemaField.useDocValuesAsStored()) {
        nonStoredDVsUsedAsStored.add(fieldInfo.name);
      }
      allNonStoredDVs.add(fieldInfo.name);
      if (!searcher.getSchema().isCopyFieldTarget(schemaField)) {
        nonStoredDVsWithoutCopyTargets.add(fieldInfo.name);
      }
    }
    if (schemaField.stored() && schemaField.isLarge()) {
      storedLargeFields.add(schemaField.getName());
    }
  }

  this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored);
  this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
  this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets);
  this.largeFields = Collections.unmodifiableSet(storedLargeFields);
  this.dvsCanSubstituteStored = Collections.unmodifiableSet(dvsCanSubstituteStored);
  this.allStored = Collections.unmodifiableSet(allStoreds);
}
 

/**
 * Converts a SolrInputDocument to SolrDocument, using an IndexSchema instance.
 *
 * @param sdoc The input document.
 * @param schema The index schema.
 * @param forInPlaceUpdate Whether the document is being used for an in place update,
 *                         see {@link DocumentBuilder#toDocument(SolrInputDocument, IndexSchema, boolean, boolean)}
 */
public static SolrDocument toSolrDoc(SolrInputDocument sdoc, IndexSchema schema, boolean forInPlaceUpdate) {
  // TODO what about child / nested docs?
  // TODO: do something more performant than this double conversion
  Document doc = DocumentBuilder.toDocument(sdoc, schema, forInPlaceUpdate, true);

  // copy the stored fields only
  Document out = new Document();
  for (IndexableField f : doc.getFields()) {
    if (f.fieldType().stored()) {
      out.add(f);
    } else if (f.fieldType().docValuesType() != DocValuesType.NONE) {
      SchemaField schemaField = schema.getFieldOrNull(f.name());
      if (schemaField != null && !schemaField.stored() && schemaField.useDocValuesAsStored()) {
        out.add(f);
      }
    } else {
      log.debug("Don't know how to handle field {}", f);
    }
  }

  SolrDocument solrDoc = toSolrDoc(out, schema);

  // add child docs
  for(SolrInputField solrInputField: sdoc) {
    if(solrInputField.getFirstValue() instanceof SolrInputDocument) {
      // is child doc
      Object val = solrInputField.getValue();
      Iterator<SolrDocument> childDocs = solrInputField.getValues().stream()
          .map(x -> toSolrDoc((SolrInputDocument) x, schema)).iterator();
      if(val instanceof Collection) {
        // add as collection even if single element collection
        solrDoc.setField(solrInputField.getName(), Lists.newArrayList(childDocs));
      } else {
        // single child doc
        solrDoc.setField(solrInputField.getName(), childDocs.next());
      }
    }
  }

  return solrDoc;
}
 

/**
 * Given a schema field, return whether or not such a field is supported for an in-place update.
 * Note: If an update command has updates to only supported fields (and _version_ is also supported),
 * only then is such an update command executed as an in-place update.
 */
public static boolean isSupportedFieldForInPlaceUpdate(SchemaField schemaField) {
  return !(schemaField.indexed() || schemaField.stored() || !schemaField.hasDocValues() || 
      schemaField.multiValued() || !(schemaField.getType() instanceof NumericValueFieldType));
}