Java Code Examples for org.apache.lucene.document.StringField#TYPE_NOT_STORED

public void update(IndexWriter writer) throws IOException {
  // Add 10 docs:
  FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  for(int j=0; j<10; j++) {
    Document d = new Document();
    int n = random().nextInt();
    d.add(newField("id", Integer.toString(nextID++), customType));
    d.add(newTextField("contents", English.intToEnglish(n), Field.Store.NO));
    writer.addDocument(d);
  }

  // Delete 5 docs:
  int deleteID = nextID-1;
  for(int j=0; j<5; j++) {
    writer.deleteDocuments(new Term("id", ""+deleteID));
    deleteID -= 2;
  }
}
 

/**
 * Creates this strategy.
 * {@code fieldType} is used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL
 * field too. Search requires pointValues (or legacy numerics), and relevancy requires docValues. If these features
 * aren't needed then disable them.
 */
public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  field_bbox = fieldNamePrefix;
  field_minX = fieldNamePrefix + SUFFIX_MINX;
  field_maxX = fieldNamePrefix + SUFFIX_MAXX;
  field_minY = fieldNamePrefix + SUFFIX_MINY;
  field_maxY = fieldNamePrefix + SUFFIX_MAXY;
  field_xdl = fieldNamePrefix + SUFFIX_XDL;

  fieldType.freeze();
  this.optionsFieldType = fieldType;

  int numQuads = 0;
  if ((this.hasStored = fieldType.stored())) {
    numQuads++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numQuads++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numQuads++;
  }

  if (hasPointVals) { // if we have an index...
    xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED);
    xdlFieldType.setIndexOptions(IndexOptions.DOCS);
    xdlFieldType.freeze();
  } else {
    xdlFieldType = null;
  }

  this.fieldsLen = numQuads * 4 + (xdlFieldType != null ? 1 : 0);
}
 

private static void addGram(String text, Document doc, int ng1, int ng2) {
  int len = text.length();
  for (int ng = ng1; ng <= ng2; ng++) {
    String key = "gram" + ng;
    String end = null;
    for (int i = 0; i < len - ng + 1; i++) {
      String gram = text.substring(i, i + ng);
      FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
      ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
      Field ngramField = new Field(key, gram, ft);
      // spellchecker does not use positional queries, but we want freqs
      // for scoring these multivalued n-gram fields.
      doc.add(ngramField);
      if (i == 0) {
        // only one term possible in the startXXField, TF/pos and norms aren't needed.
        Field startField = new StringField("start" + ng, gram, Field.Store.NO);
        doc.add(startField);
      }
      end = gram;
    }
    if (end != null) { // may not be present if len==ng1
      // only one term possible in the endXXField, TF/pos and norms aren't needed.
      Field endField = new StringField("end" + ng, end, Field.Store.NO);
      doc.add(endField);
    }
  }
}
 

@Override
public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) {
  String name = getName(family, column.getName(), subName);
  Field field = new Field(name, column.getValue(), StringField.TYPE_NOT_STORED);
  if (isSortEnable()) {
    return addSort(column, name, field);
  }
  return makeIterable(field);
}
 

@Override
public IndexableFieldType fieldType() {
  return StringField.TYPE_NOT_STORED;
}
 

public void testDoubleOffsetCounting() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(true);
  customType.setStoreTermVectorOffsets(true);
  Field f = newField("field", "abcd", customType);
  doc.add(f);
  doc.add(f);
  Field f2 = newField("field", "", customType);
  doc.add(f2);
  doc.add(f);
  w.addDocument(doc);
  w.close();

  IndexReader r = DirectoryReader.open(dir);
  Terms vector = r.getTermVectors(0).terms("field");
  assertNotNull(vector);
  TermsEnum termsEnum = vector.iterator();
  assertNotNull(termsEnum.next());
  assertEquals("", termsEnum.term().utf8ToString());

  // Token "" occurred once
  assertEquals(1, termsEnum.totalTermFreq());

  PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL);
  assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
  dpEnum.nextPosition();
  assertEquals(8, dpEnum.startOffset());
  assertEquals(8, dpEnum.endOffset());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc());

  // Token "abcd" occurred three times
  assertEquals(new BytesRef("abcd"), termsEnum.next());
  dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL);
  assertEquals(3, termsEnum.totalTermFreq());

  assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
  dpEnum.nextPosition();
  assertEquals(0, dpEnum.startOffset());
  assertEquals(4, dpEnum.endOffset());

  dpEnum.nextPosition();
  assertEquals(4, dpEnum.startOffset());
  assertEquals(8, dpEnum.endOffset());

  dpEnum.nextPosition();
  assertEquals(8, dpEnum.startOffset());
  assertEquals(12, dpEnum.endOffset());

  assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc());
  assertNull(termsEnum.next());
  r.close();
  dir.close();
}
 

public void testTermVectorCorruption3() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
      .setMaxBufferedDocs(2)
      .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setMergeScheduler(new SerialMergeScheduler())
      .setMergePolicy(new LogDocMergePolicy()));

  Document document = new Document();
  FieldType customType = new FieldType();
  customType.setStored(true);

  Field storedField = newField("stored", "stored", customType);
  document.add(storedField);
  FieldType customType2 = new FieldType(StringField.TYPE_NOT_STORED);
  customType2.setStoreTermVectors(true);
  customType2.setStoreTermVectorPositions(true);
  customType2.setStoreTermVectorOffsets(true);
  Field termVectorField = newField("termVector", "termVector", customType2);
  document.add(termVectorField);
  for(int i=0;i<10;i++)
    writer.addDocument(document);
  writer.close();

  writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
      .setMaxBufferedDocs(2)
      .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setMergeScheduler(new SerialMergeScheduler())
      .setMergePolicy(new LogDocMergePolicy()));
  for(int i=0;i<6;i++)
    writer.addDocument(document);

  writer.forceMerge(1);
  writer.close();

  IndexReader reader = DirectoryReader.open(dir);
  for(int i=0;i<10;i++) {
    reader.getTermVectors(i);
    reader.document(i);
  }
  reader.close();
  dir.close();
}
 

/**
 * Creates this strategy.
 * {@code fieldType} is used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL
 * field too. Search requires pointValues (or legacy numerics), and relevancy requires docValues. If these features
 * aren't needed then disable them.
 */
public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
  super(ctx, fieldNamePrefix);
  field_bbox = fieldNamePrefix;
  field_minX = fieldNamePrefix + SUFFIX_MINX;
  field_maxX = fieldNamePrefix + SUFFIX_MAXX;
  field_minY = fieldNamePrefix + SUFFIX_MINY;
  field_maxY = fieldNamePrefix + SUFFIX_MAXY;
  field_xdl = fieldNamePrefix + SUFFIX_XDL;

  fieldType.freeze();
  this.optionsFieldType = fieldType;

  int numQuads = 0;
  if ((this.hasStored = fieldType.stored())) {
    numQuads++;
  }
  if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
    numQuads++;
  }
  if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
    numQuads++;
  }
  if (fieldType.indexOptions() != IndexOptions.NONE && fieldType instanceof LegacyFieldType && ((LegacyFieldType)fieldType).numericType() != null) {
    if (hasPointVals) {
      throw new IllegalArgumentException("pointValues and LegacyNumericType are mutually exclusive");
    }
    final LegacyFieldType legacyType = (LegacyFieldType) fieldType;
    if (legacyType.numericType() != LegacyNumericType.DOUBLE) {
      throw new IllegalArgumentException(getClass() + " does not support " + legacyType.numericType());
    }
    numQuads++;
    legacyNumericFieldType = new LegacyFieldType(LegacyDoubleField.TYPE_NOT_STORED);
    legacyNumericFieldType.setNumericPrecisionStep(legacyType.numericPrecisionStep());
    legacyNumericFieldType.freeze();
  } else {
    legacyNumericFieldType = null;
  }

  if (hasPointVals || legacyNumericFieldType != null) { // if we have an index...
    xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED);
    xdlFieldType.setIndexOptions(IndexOptions.DOCS);
    xdlFieldType.freeze();
  } else {
    xdlFieldType = null;
  }

  this.fieldsLen = numQuads * 4 + (xdlFieldType != null ? 1 : 0);
}