org.apache.lucene.document.StoredField Java Examples

@Before
public void setUp() throws Exception {
    analyzer = new StandardAnalyzer();
    tempDirectory = Files.createTempDirectory("lucene");
    directory = new MMapDirectory(tempDirectory);
    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    IndexWriter iwriter = new IndexWriter(directory, config);

    Document doc = new Document();
    doc.add(new Field("contents", "name=text", TextField.TYPE_STORED));

    IntPoint intPoint = new IntPoint("intfield", 4);
    doc.add(intPoint);
    doc.add(new StoredField("intfield", 4));
    iwriter.addDocument(doc);

    iwriter.close();
    ireader = DirectoryReader.open(directory);
    isearcher = new IndexSearcher(ireader);
}
 

protected static ParsedDocument testParsedDocument(
    String id, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate,
    boolean recoverySource) {
    Field uidField = new Field("_id", Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE);
    Field versionField = new NumericDocValuesField("_version", 0);
    SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
    document.add(uidField);
    document.add(versionField);
    document.add(seqID.seqNo);
    document.add(seqID.seqNoDocValue);
    document.add(seqID.primaryTerm);
    BytesRef ref = source.toBytesRef();
    if (recoverySource) {
        document.add(new StoredField(SourceFieldMapper.RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
        document.add(new NumericDocValuesField(SourceFieldMapper.RECOVERY_SOURCE_NAME, 1));
    } else {
        document.add(new StoredField(SourceFieldMapper.NAME, ref.bytes, ref.offset, ref.length));
    }
    return new ParsedDocument(versionField, seqID, id, routing, Arrays.asList(document), source, mappingUpdate);
}
 

private void addToDoc(IndexKey key, Double value){
  Preconditions.checkArgument(key.getValueType() == Double.class);
  if(value == null){
    return;
  }

  checkIfMultiValueField(key);

  final String indexFieldName = key.getIndexFieldName();
  doc.add(new DoublePoint(indexFieldName, value));
  if (key.isStored()) {
    doc.add(new StoredField(indexFieldName, value));
  }
  if (key.isSorted()) {
    Preconditions.checkArgument(key.getSortedValueType() == SearchFieldSorting.FieldType.DOUBLE);
    doc.add(new DoubleDocValuesField(indexFieldName, value));
  }
}
 

/** test a null byte[] value doesn't abort the entire segment */
public void testNullStoredBytesFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", new BytesRef("hello").bytes);
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(NullPointerException.class, () -> {
    // set to null value
    byte v[] = null;
    theField.setBytesValue(v);
    iw.addDocument(doc);
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 

/** test a null string value doesn't abort the entire segment */
public void testNullStoredFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", "hello", StoredField.TYPE);
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    theField.setStringValue(null);
    iw.addDocument(doc);
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 

/** test a null bytesref value doesn't abort the entire segment */
public void testNullStoredBytesRefField() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  iw.addDocument(doc);

  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    BytesRef v = null;
    Field theField = new StoredField("foo", v);
    doc.add(theField);
    iw.addDocument(doc);
    fail("didn't get expected exception");
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 

void addToDoc(IndexKey key, Integer value){
  Preconditions.checkArgument(key.getValueType() == Integer.class);
  if(value == null){
    return;
  }

  checkIfMultiValueField(key);

  final String indexFieldName = key.getIndexFieldName();
  doc.add(new IntPoint(indexFieldName, value));
  if (key.isStored()) {
    doc.add(new StoredField(indexFieldName, value));
  }
  if (key.isSorted()) {
    Preconditions.checkArgument(key.getSortedValueType() == SearchFieldSorting.FieldType.INTEGER);
    doc.add(new NumericDocValuesField(indexFieldName, value));
  }
}
 

/** test a null bytesref value doesn't abort the entire segment */
public void testNullStoredBytesRefFieldReuse() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  Field theField = new StoredField("foo", new BytesRef("hello"));
  doc.add(theField);
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    BytesRef v = null;
    theField.setBytesValue(v);
    iw.addDocument(doc);
    fail("didn't get expected exception");
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 

/** test a null string value doesn't abort the entire segment */
public void testNullStoredField() throws Exception {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));
  // add good document
  Document doc = new Document();
  iw.addDocument(doc);
  expectThrows(IllegalArgumentException.class, () -> {
    // set to null value
    String value = null;
    doc.add(new StoredField("foo", value));
    iw.addDocument(doc);
  });

  assertNull(iw.getTragicException());
  iw.close();
  // make sure we see our good doc
  DirectoryReader r = DirectoryReader.open(dir);
  assertEquals(1, r.numDocs());
  r.close();
  dir.close();
}
 

@Override
public NavigableMap<String, IndexableField> encode(LuceneContext context, String path, Field field, LuceneStore annotation, Type type, Object instance) {
    NavigableMap<String, IndexableField> indexables = new TreeMap<>();
    Class<?> componentClass = null;
    Type componentType = null;
    if (type instanceof GenericArrayType) {
        GenericArrayType genericArrayType = GenericArrayType.class.cast(type);
        componentType = genericArrayType.getGenericComponentType();
        componentClass = TypeUtility.getRawType(componentType, null);
    } else {
        Class<?> clazz = TypeUtility.getRawType(type, null);
        componentType = clazz.getComponentType();
        componentClass = clazz.getComponentType();
    }
    Specification specification = Specification.getSpecification(componentClass);
    StoreConverter converter = context.getStoreConverter(specification);
    int size = Array.getLength(instance);
    IndexableField indexable = new StoredField(path + ".size", size);
    indexables.put(path + ".size", indexable);
    for (int index = 0; index < size; index++) {
        Object element = Array.get(instance, index);
        indexables.putAll(converter.encode(context, path + "[" + index + "]", field, annotation, componentType, element));
    }
    return indexables;
}
 

@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  String externalVal = value.toString();
  String[] point = parseCommaSeparatedList(externalVal, dimension);

  // TODO: this doesn't currently support polyFields as sub-field types
  List<IndexableField> f = new ArrayList<>((dimension*2)+1);

  if (field.indexed()) {
    for (int i=0; i<dimension; i++) {
      SchemaField sf = subField(field, i, schema);
      f.addAll(sf.createFields(point[i]));
    }
  }

  if (field.stored()) {
    String storedVal = externalVal;  // normalize or not?
    f.add(createField(field.getName(), storedVal, StoredField.TYPE));
  }

  return f;
}
 

public static Document Document(String inLine, int lineNumber) {
	  	
		Document doc = new Document();
	    doc.add(new StoredField("line_number", ""+lineNumber));
	    doc.add(new StoredField("modified",
	                      DateTools.timeToString(System.currentTimeMillis(), DateTools.Resolution.MINUTE)));
	    
	    //create new FieldType to store term positions (TextField is not sufficiently configurable)
	    FieldType ft = new FieldType();
	    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
	    ft.setTokenized(true);
	    ft.setStoreTermVectors(true);
	    ft.setStoreTermVectorPositions(true);
	    Field contentsField = new Field("contents", inLine, ft);

	    doc.add(contentsField);
	    return doc;
}
 

public void testNoAbortOnBadTVSettings() throws Exception {
  Directory dir = newDirectory();
  // Don't use RandomIndexWriter because we want to be sure both docs go to 1 seg:
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter iw = new IndexWriter(dir, iwc);

  Document doc = new Document();
  iw.addDocument(doc);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectors(true);
  ft.freeze();
  doc.add(new Field("field", "value", ft));

  expectThrows(IllegalArgumentException.class, () -> {
    iw.addDocument(doc);
  });

  IndexReader r = DirectoryReader.open(iw);

  // Make sure the exc didn't lose our first document:
  assertEquals(1, r.numDocs());
  iw.close();
  r.close();
  dir.close();
}
 

private Document newGeoDocument(OIdentifiable oIdentifiable, Shape shape) {

    FieldType ft = new FieldType();
    ft.setIndexed(true);
    ft.setStored(true);

    Document doc = new Document();

    doc.add(OLuceneIndexType.createField(RID, oIdentifiable.getIdentity().toString(), Field.Store.YES,
        Field.Index.NOT_ANALYZED_NO_NORMS));
    for (IndexableField f : strategy.createIndexableFields(shape)) {
      doc.add(f);
    }

    doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));

    return doc;
  }
 

public void testIllegalVectorPayloadsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorPayloads(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector payloads for a field that is not indexed (field=\"field\")", expected.getMessage());

  w.close();
  dir.close();
}
 

public void testBulkMergeWithDeletes() throws IOException {
  final int numDocs = atLeast(200);
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
  for (int i = 0; i < numDocs; ++i) {
    Document doc = new Document();
    doc.add(new StringField("id", Integer.toString(i), Store.YES));
    doc.add(new StoredField("f", TestUtil.randomSimpleString(random())));
    w.addDocument(doc);
  }
  final int deleteCount = TestUtil.nextInt(random(), 5, numDocs);
  for (int i = 0; i < deleteCount; ++i) {
    final int id = random().nextInt(numDocs);
    w.deleteDocuments(new Term("id", Integer.toString(id)));
  }
  w.commit();
  w.close();
  w = new RandomIndexWriter(random(), dir);
  w.forceMerge(TestUtil.nextInt(random(), 1, 3));
  w.commit();
  w.close();
  TestUtil.checkIndex(dir);
  dir.close();
}
 

@Override
public List<Field> createFields(String name, Number value,
                                boolean indexed, boolean docValued, boolean stored) {
    List<Field> fields = new ArrayList<>();
    if (indexed) {
        fields.add(new DoublePoint(name, value.doubleValue()));
    }
    if (docValued) {
        fields.add(new SortedNumericDocValuesField(name,
            NumericUtils.doubleToSortableLong(value.doubleValue())));
    }
    if (stored) {
        fields.add(new StoredField(name, value.doubleValue()));
    }
    return fields;
}
 

@Override
public List<IndexableField> createFields(SchemaField field, Object externalVal) {
  CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);

  List<IndexableField> f = new ArrayList<>();
  SchemaField amountField = getAmountField(field);
  f.add(amountField.createField(String.valueOf(value.getAmount())));
  SchemaField currencyField = getCurrencyField(field);
  f.add(currencyField.createField(value.getCurrencyCode()));

  if (field.stored()) {
    String storedValue = externalVal.toString().trim();
    if (storedValue.indexOf(",") < 0) {
      storedValue += "," + defaultCurrency;
    }
    f.add(createField(field.getName(), storedValue, StoredField.TYPE));
  }

  return f;
}
 

@Override
public Iterable<? extends Field> getFieldsForColumn(String family, Column column) {
  synchronized (_strategy) {
    String name = getName(family, column.getName());
    if (!_strategy.getFieldName().equals(name)) {
      throw new RuntimeException("Strategy name and column name do not match.");
    }
    List<Field> fields = new ArrayList<Field>();
    Shape shape = getShape(column);
    checkShape(shape);
    for (Field f : _strategy.createIndexableFields(shape)) {
      fields.add(f);
    }
    fields.add(new StoredField(name, column.getValue()));
    return fields;
  }
}
 

public void testIllegalVectorPositionsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorPositions(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector positions for a field that is not indexed (field=\"field\")", expected.getMessage());
  
  w.close();
  dir.close();
}
 

protected List<Document> getDocuments(Iterator<SpatialTestData> sampleData) {
  List<Document> documents = new ArrayList<>();
  while (sampleData.hasNext()) {
    SpatialTestData data = sampleData.next();
    Document document = new Document();
    document.add(new StringField("id", data.id, Field.Store.YES));
    document.add(new StringField("name", data.name, Field.Store.YES));
    Shape shape = data.shape;
    shape = convertShapeFromGetDocuments(shape);
    if (shape != null) {
      for (Field f : strategy.createIndexableFields(shape)) {
        document.add(f);
      }
      if (storeShape)//just for diagnostics
        document.add(new StoredField(strategy.getFieldName(), shape.toString()));
    }

    documents.add(document);
  }
  return documents;
}
 

private Document newSampleDocument(int id, Shape... shapes) {
  Document doc = new Document();
  doc.add(new StoredField("id", id));
  doc.add(new NumericDocValuesField("id", id));
  //Potentially more than one shape in this field is supported by some
  // strategies; see the javadocs of the SpatialStrategy impl to see.
  for (Shape shape : shapes) {
    for (Field f : strategy.createIndexableFields(shape)) {
      doc.add(f);
    }
    //store it too; the format is up to you
    //  (assume point in this example)
    Point pt = (Point) shape;
    doc.add(new StoredField(strategy.getFieldName(), pt.getX()+" "+pt.getY()));
  }

  return doc;
}
 

public void testIllegalVectorOffsetsWithoutIndexed() throws Exception {
  Directory dir = newDirectory();
  MockAnalyzer a = new MockAnalyzer(random());
  a.setEnableChecks(false);
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, a);
  FieldType ft = new FieldType(StoredField.TYPE);
  ft.setStoreTermVectorOffsets(true);
  Document doc = new Document();
  doc.add(new Field("field", "value", ft));
  
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });
  assertEquals("cannot store term vector offsets for a field that is not indexed (field=\"field\")", expected.getMessage());
  
  w.close();
  dir.close();
}
 

@Override
protected Document newDoc(String id, Shape shape) {
  Document doc = new Document();
  doc.add(new StringField("id", id, Field.Store.YES));
  if (shape != null) {
    Collection<Shape> shapes;
    if (shape instanceof ShapePair) {
      shapes = new ArrayList<>(2);
      shapes.add(((ShapePair)shape).shape1);
      shapes.add(((ShapePair)shape).shape2);
    } else {
      shapes = Collections.singleton(shape);
    }
    for (Shape shapei : shapes) {
      for (Field f : strategy.createIndexableFields(shapei)) {
        doc.add(f);
      }
    }
    if (storeShape)//just for diagnostics
      doc.add(new StoredField(strategy.getFieldName(), shape.toString()));
  }
  return doc;
}
 

public Document prepareDocumentForFwdIndex(Bag bag) {
    Document document = new Document();
    StringField idField = new StringField("id", bag.getId() + "",
            Field.Store.NO);
    //idField.fieldType().setIndexed(true);
    //idField.fieldType().freeze();
    document.add(idField);
    
    StringBuilder tokenString = new StringBuilder();
    for (TokenFrequency tf : bag) {
        // System.out.println(tf.getToken().getValue() +
        // ":"+tf.getFrequency());
        tokenString.append(tf.getToken().getValue() + ":" + tf.getFrequency() + "::");
    }
    StoredField strField = new StoredField("tokens", tokenString.toString().trim());
    document.add(strField);
    return document;
}
 

@Override
public void addDoc(Object document, int docIdCounter) {
  Document docToIndex = new Document();
  docToIndex.add(new TextField(_textColumn, document.toString(), Field.Store.NO));
  docToIndex.add(new StoredField(LUCENE_INDEX_DOC_ID_COLUMN_NAME, docIdCounter));
  try {
    _indexWriter.addDocument(docToIndex);
  } catch (Exception e) {
    LOGGER.error("Failure while adding a new document to index for column {}, exception {}", _textColumn, e.getMessage());
    throw new RuntimeException(e);
  }
}
 

@Override
protected void addRandomFields(Document d) {
  final int numValues = random().nextInt(3);
  for (int i = 0; i < numValues; ++i) {
    d.add(new StoredField("f", TestUtil.randomSimpleString(random(), 100)));
  }
}
 

public void testBinaryFieldOffsetLength() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  byte[] b = new byte[50];
  for(int i=0;i<50;i++)
    b[i] = (byte) (i+77);

  Document doc = new Document();
  Field f = new StoredField("binary", b, 10, 17);
  byte[] bx = f.binaryValue().bytes;
  assertTrue(bx != null);
  assertEquals(50, bx.length);
  assertEquals(10, f.binaryValue().offset);
  assertEquals(17, f.binaryValue().length);
  doc.add(f);
  w.addDocument(doc);
  w.close();

  IndexReader ir = DirectoryReader.open(dir);
  Document doc2 = ir.document(0);
  IndexableField f2 = doc2.getField("binary");
  b = f2.binaryValue().bytes;
  assertTrue(b != null);
  assertEquals(17, b.length, 17);
  assertEquals(87, b[0]);
  ir.close();
  dir.close();
}
 

@Ignore("requires running tests with biggish heap")
public void testMassiveField() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  final IndexWriter w = new IndexWriter(dir, iwc);

  StringBuilder b = new StringBuilder();
  while (b.length() <= IndexWriter.MAX_STORED_STRING_LENGTH) {
    b.append("x ");
  }

  final Document doc = new Document();
  //doc.add(new TextField("big", b.toString(), Field.Store.YES));
  doc.add(new StoredField("big", b.toString()));
  Exception e = expectThrows(IllegalArgumentException.class, () -> {w.addDocument(doc);});
  assertEquals("stored field \"big\" is too large (" + b.length() + " characters) to store", e.getMessage());

  // make sure writer is still usable:
  Document doc2 = new Document();
  doc2.add(new StringField("id", "foo", Field.Store.YES));
  w.addDocument(doc2);

  DirectoryReader r = DirectoryReader.open(w);
  assertEquals(1, r.numDocs());
  r.close();
  w.close();
  dir.close();
}
 

@Override
public List<Field> createFields(String name, Number value,
                                boolean indexed, boolean docValued, boolean stored) {
    List<Field> fields = new ArrayList<>();
    if (indexed) {
        fields.add(new LongPoint(name, value.longValue()));
    }
    if (docValued) {
        fields.add(new SortedNumericDocValuesField(name, value.longValue()));
    }
    if (stored) {
        fields.add(new StoredField(name, value.longValue()));
    }
    return fields;
}