org.apache.lucene.index.IndexableField Java Examples

protected IndexableField buildFacetField(String f, Object val) {
  String[] path = null;
  if (val instanceof String) {

    path = ((String) val).split("/");
    // path = new String[1];
    // path[0] = (String) val;
  } else if (val instanceof Iterable) {
    Iterable iterable = (Iterable) val;
    List<String> values = new ArrayList<String>();
    for (Object s : iterable) {
      if (s instanceof String) {
        values.add((String) s);
      } else {
        throw new OIndexEngineException("Cannot facet value " + val + " because it is not a string", null);
      }
    }
    path = values.toArray(new String[values.size()]);
  }
  return new FacetField(f, path);
}
 

@Override
public NavigableMap<String, IndexableField> encode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, Object instance) {
    NavigableMap<String, IndexableField> indexables = new TreeMap<>();
    Class<?> componentClass = null;
    Type componentType = null;
    if (type instanceof GenericArrayType) {
        GenericArrayType genericArrayType = GenericArrayType.class.cast(type);
        componentType = genericArrayType.getGenericComponentType();
        componentClass = TypeUtility.getRawType(componentType, null);
    } else {
        Class<?> clazz = TypeUtility.getRawType(type, null);
        componentType = clazz.getComponentType();
        componentClass = clazz.getComponentType();
    }
    Specification specification = Specification.getSpecification(componentClass);
    StoreConverter converter = context.getStoreConverter(specification);
    int size = Array.getLength(instance);
    IndexableField indexable = new StoredField(path + ".size", size);
    indexables.put(path + ".size", indexable);
    for (int index = 0; index < size; index++) {
        Object element = Array.get(instance, index);
        indexables.putAll(converter.encode(context, path + "[" + index + "]", field, annotation, componentType, element));
    }
    return indexables;
}
 

/**
 * Creates a StorableField whose value will be lazy loaded if and 
 * when it is used. 
 * <p>
 * <b>NOTE:</b> This method must be called once for each value of the field 
 * name specified in sequence that the values exist.  This method may not be 
 * used to generate multiple, lazy, StorableField instances referring to 
 * the same underlying StorableField instance.
 * </p>
 * <p>
 * The lazy loading of field values from all instances of StorableField 
 * objects returned by this method are all backed by a single StoredDocument 
 * per LazyDocument instance.
 * </p>
 */
public IndexableField getField(FieldInfo fieldInfo) {  

  fieldNames.add(fieldInfo.name);
  List<LazyField> values = fields.get(fieldInfo.number);
  if (null == values) {
    values = new ArrayList<>();
    fields.put(fieldInfo.number, values);
  } 

  LazyField value = new LazyField(fieldInfo.name, fieldInfo.number);
  values.add(value);

  synchronized (this) {
    // edge case: if someone asks this LazyDoc for more LazyFields
    // after other LazyFields from the same LazyDoc have been
    // actuallized, we need to force the doc to be re-fetched
    // so the new LazyFields are also populated.
    doc = null;
  }
  return value;
}
 

private ProvenanceEventRecord getRecord(final Document d, final RecordReader reader) throws IOException {
    final IndexableField blockField = d.getField(FieldNames.BLOCK_INDEX);
    if ( blockField == null ) {
        reader.skipTo(getByteOffset(d, reader));
    } else {
        reader.skipToBlock(blockField.numericValue().intValue());
    }

    StandardProvenanceEventRecord record;
    while ( (record = reader.nextRecord()) != null) {
        final IndexableField idField = d.getField(SearchableFields.Identifier.getSearchableFieldName());
        if ( idField == null || idField.numericValue().longValue() == record.getEventId() ) {
            break;
        }
    }

    if (record == null) {
        logger.warn("Failed to read Provenance Event for '" + d + "'. The event file may be missing or corrupted");
    }

    return record;
}
 

@Override
protected void updateDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
  final long gen = genWriter.updateDocument(id, doc);
  // Randomly verify the udpate "took":
  if (random().nextInt(20) == 2) {
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: verify updateDocument " + id + " gen=" + gen);
    }
    nrtDeletesThread.waitForGeneration(gen);
    assertTrue(gen <= nrtDeletesThread.getSearchingGen());
    final IndexSearcher s = nrtDeletes.acquire();
    if (VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": nrt: got deletes searcher=" + s);
    }
    try {
      assertEquals("generation: " + gen, 1, s.search(new TermQuery(id), 10).totalHits.value);
    } finally {
      nrtDeletes.release(s);
    }
  }
  lastGens.set(gen);
}
 

private void assertData(int totalShardCount) throws IOException {
  Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
  for (int i = 0; i < totalShardCount; i++) {
    HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, ShardUtil.getShardName(i)));
    DirectoryReader reader = DirectoryReader.open(directory);
    int numDocs = reader.numDocs();
    for (int d = 0; d < numDocs; d++) {
      Document document = reader.document(d);
      IndexableField field = document.getField("id");
      Integer id = (Integer) field.numericValue();
      int partition = partitioner.getPartition(new IntWritable(id), null, totalShardCount);
      assertEquals(i, partition);
    }
    reader.close();
  }
}
 

@Override
public List<IndexableField> createFields(SchemaField field, Object externalVal) {
  CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);

  List<IndexableField> f = new ArrayList<>();
  SchemaField amountField = getAmountField(field);
  f.add(amountField.createField(String.valueOf(value.getAmount())));
  SchemaField currencyField = getCurrencyField(field);
  f.add(currencyField.createField(value.getCurrencyCode()));

  if (field.stored()) {
    String storedValue = externalVal.toString().trim();
    if (storedValue.indexOf(",") < 0) {
      storedValue += "," + defaultCurrency;
    }
    f.add(createField(field.getName(), storedValue, StoredField.TYPE));
  }

  return f;
}
 

@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 

public void testWhiteboxCreateFields() throws Exception {
  List<IndexableField> values = null;

  // common case...
  for (String field : Arrays.asList("keyword_stxt", "keyword_dv_stxt",
                                    "whitespace_stxt", "whitespace_f_stxt", "whitespace_l_stxt")) {
    values = createIndexableFields(field);
    assertEquals(field, 2, values.size());
    assertThat(field, values.get(0), instanceOf(Field.class));
    assertThat(field, values.get(1), instanceOf(SortedDocValuesField.class));
  }
  
  // special cases...
  values = createIndexableFields("whitespace_nois_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(SortedDocValuesField.class));
  //
  values = createIndexableFields("whitespace_nodv_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  //
  values = createIndexableFields("whitespace_m_stxt");
  assertEquals(2, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  assertThat(values.get(1), instanceOf(SortedSetDocValuesField.class));      
}
 

private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 

private Iterable<? extends IndexableField> getDoc(int docId, String read, String discover, String field1,
    String field2) {
  Document doc = new Document();
  doc.add(new StringField("id", Integer.toString(docId), Store.YES));
  AccessControlWriter writer = _accessControlFactory.getWriter();
  doc.add(new StringField("f1", field1, Store.YES));
  doc.add(new StringField("f2", field2, Store.YES));
  doc.add(new TextField("text", "constant text", Store.YES));
  Iterable<? extends IndexableField> fields = doc;
  if (read != null) {
    fields = writer.addReadVisiblity(read, doc);
  }
  if (discover != null) {
    fields = writer.addDiscoverVisiblity(discover, fields);
  }
  return fields;
}
 

private static FieldType getFieldTypeNotStored(IndexableField indexableField) {
  Field field = (Field) indexableField;
  FieldType fieldType = field.fieldType();
  FieldType result = new FieldType(fieldType);
  result.setStored(false);
  result.freeze();
  return result;
}
 

/**
 * Ensure we don't have {@link org.apache.lucene.document.LazyDocument.LazyField} or equivalent.
 * It can pose problems if the searcher is about to be closed and we haven't fetched a value yet.
 */
private static IndexableField materialize(IndexableField in) {
  if (in instanceof Field) { // already materialized
    return in;
  }
  return new ClonedField(in);
}
 

protected String convertFieldValue(Object val) {
  
  if (val instanceof IndexableField) {
    IndexableField f = (IndexableField)val;
    return f.stringValue();
  }
  return val.toString();
  
}
 

protected void doTestIntContentMatchWithQuery(Query query) throws Exception {

        ScoreDoc[] hits = isearcher.search(query, 1000).scoreDocs;
        assertEquals(1, hits.length);
        // Iterate through the results:
        for (int i = 0; i < hits.length; i++) {
            Document hitDoc = isearcher.doc(hits[i].doc);
            IndexableField field = hitDoc.getField("intfield");
            assertEquals(4, field.numericValue().intValue());
        }

    }
 

@Override
public void write(String name, TextWriter writer) throws IOException {
  String str = null;
  if(val instanceof IndexableField) { // delays holding it in memory
    str = ((IndexableField)val).stringValue();
  }
  else {
    str = val.toString();
  }
  writer.getWriter().write(str);
}
 

/** Like {@link #printableUniqueKey(org.apache.lucene.document.Document)} */
public String printableUniqueKey(SolrDocument solrDoc) {
  Object val = solrDoc.getFieldValue(uniqueKeyFieldName);
  if (val == null) {
    return null;
  } else if (val instanceof IndexableField) {
    return uniqueKeyFieldType.toExternal((IndexableField) val);
  } else {
    return val.toString();
  }
}
 

private String getFieldValueString(SolrDocument doc, String fieldName)
{
    IndexableField field = (IndexableField)doc.getFieldValue(fieldName);
    String value = null;
    if (field != null)
    {
        value = field.stringValue();
    }
    return value;
}
 

@Override
public Object decode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, NavigableMap<String, IndexableField> indexables) {
    String from = path;
    char character = path.charAt(path.length() - 1);
    character++;
    String to = path.substring(0, path.length() - 1) + character;
    indexables = indexables.subMap(from, true, to, false);
    Class<?> clazz = TypeUtility.getRawType(type, null);
    // 兼容UniMi
    type = TypeUtility.refineType(type, Map.class);
    ParameterizedType parameterizedType = ParameterizedType.class.cast(type);
    Type[] types = parameterizedType.getActualTypeArguments();
    Type keyType = types[0];
    Class<?> keyClazz = TypeUtility.getRawType(keyType, null);
    Type valueType = types[1];
    Class<?> valueClazz = TypeUtility.getRawType(valueType, null);

    try {
        // TODO 此处需要代码重构
        Map<Object, Object> map = (Map) context.getInstance(clazz);
        Specification keySpecification = Specification.getSpecification(keyClazz);
        StoreConverter keyConverter = context.getStoreConverter(keySpecification);
        Specification valueSpecification = Specification.getSpecification(valueClazz);
        StoreConverter valueConverter = context.getStoreConverter(valueSpecification);

        IndexableField indexable = indexables.get(path + ".size");
        int size = indexable.numericValue().intValue();
        for (int index = 0; index < size; index++) {
            Object key = keyConverter.decode(context, path + "[" + index + "_key]", field, annotation, keyType, indexables);
            Object value = valueConverter.decode(context, path + "[" + index + "_value]", field, annotation, valueType, indexables);
            map.put(key, value);
        }
        return map;
    } catch (Exception exception) {
        // TODO
        throw new StorageException(exception);
    }
}
 

private void assertLazyNotLoaded(Document d, String fieldName) {
  IndexableField field = d.getField(fieldName);
  if (fieldName == BIG_FIELD) {
    assertTrue(field instanceof SolrDocumentFetcher.LargeLazyField);
    assertFalse(((SolrDocumentFetcher.LargeLazyField)field).hasBeenLoaded());
  } else {
    assertTrue(field instanceof LazyDocument.LazyField);
    assertFalse(((LazyDocument.LazyField)field).hasBeenLoaded());
  }
}
 

private static boolean isDocValue(IndexableField field) {
  if (field instanceof BinaryDocValuesField) {
    return true;
  } else if (field instanceof NumericDocValuesField) {
    return true;
  } else if (field instanceof SortedDocValuesField) {
    return true;
  } else if (field instanceof SortedSetDocValuesField) {
    return true;
  } else {
    return false;
  }
}
 

/** test that Document.getFields() actually returns an immutable list */
public void testGetFieldsImmutable() {
  Document doc = makeDocumentWithFields();
  assertEquals(10, doc.getFields().size());
  List<IndexableField> fields = doc.getFields();
  expectThrows(UnsupportedOperationException.class, () -> {
    fields.add(new StringField("name", "value", Field.Store.NO));
  });

  expectThrows(UnsupportedOperationException.class, () -> {
    fields.clear();
  });
}
 

protected void createFieldNamesField(ParseContext context, List<IndexableField> fields) {
    FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context.docMapper()
            .metadataMapper(FieldNamesFieldMapper.class).fieldType();
    if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) {
        for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) {
            fields.add(new Field(FieldNamesFieldMapper.NAME, fieldName, fieldNamesFieldType));
        }
    }
}
 

private static Set<String> getFieldsToMask(Iterable<? extends IndexableField> fields) {
  Set<String> result = new HashSet<String>();
  for (IndexableField field : fields) {
    if (field.name().equals(READ_MASK_FIELD)) {
      result.add(getFieldNameOnly(field.stringValue()));
    }
  }
  return result;
}
 

@Override
public Set<Long> getErrorDocIds() throws IOException
{
    Set<Long> errorDocIds = new HashSet<>();
    RefCounted<SolrIndexSearcher> refCounted = null;
    try
    {
        refCounted = this.core.getSearcher();
        SolrIndexSearcher searcher = refCounted.get();
        TermQuery errorQuery = new TermQuery(new Term(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE));
        DocListCollector docListCollector = new DocListCollector();
        searcher.search(errorQuery, docListCollector);
        IntArrayList docList = docListCollector.getDocs();
        int size = docList.size();

        for (int i = 0; i < size; ++i)
        {
            int doc = docList.get(i);
            Document document = searcher.doc(doc, REQUEST_ONLY_ID_FIELD);
            IndexableField id = document.getField(FIELD_SOLR4_ID);
            String idString = id.stringValue();

            if (idString.startsWith(PREFIX_ERROR))
            {
                idString = idString.substring(PREFIX_ERROR.length());
            }

            errorDocIds.add(Long.valueOf(idString));
        }
    }
    finally
    {
        ofNullable(refCounted).ifPresent(RefCounted::decref);
    }
    return errorDocIds;
}
 

@Override
public IndexableField createField(SchemaField field, Object value) {
  Date date = (value instanceof Date)
      ? ((Date)value)
      : DateMathParser.parseMath(null, value.toString());
  return new LongPoint(field.getName(), date.getTime());
}
 

public BytesRef getBinaryValue(String name) {
    for (IndexableField f : fields) {
        if (f.name().equals(name) && f.binaryValue() != null) {
            return f.binaryValue();
        }
    }
    return null;
}
 

private static List<IndexableField> getIndexableFields(SchemaField field, IndexableField f, BytesRef bytes) {
  final IndexableField docval = field.multiValued()
    ? new SortedSetDocValuesField(field.getName(), bytes)
    : new SortedDocValuesField(field.getName(), bytes);
  
  if (null == f) {
    return Collections.singletonList(docval);
  } 
  return Arrays.asList(f, docval);
}
 

@Override
public List<IndexableField> createFields(SchemaField field, Object val) {
  String shapeStr = null;
  Shape shape;
  if (val instanceof Shape) {
    shape = ((Shape) val);
  } else {
    shapeStr = val.toString();
    shape = parseShape(shapeStr);
  }
  if (shape == null) {
    log.debug("Field {}: null shape for input: {}", field, val);
    return Collections.emptyList();
  }

  List<IndexableField> result = new ArrayList<>();
  if (field.indexed() || field.hasDocValues()) {
    T strategy = getStrategy(field.getName());
    result.addAll(Arrays.asList(strategy.createIndexableFields(shape)));
  }

  if (field.stored()) {
    result.add(new StoredField(field.getName(), getStoredValue(shape, shapeStr)));
  }

  return result;
}
 

public String get(String name) {
    for (IndexableField f : fields) {
        if (f.name().equals(name) && f.stringValue() != null) {
            return f.stringValue();
        }
    }
    return null;
}