org.apache.lucene.codecs.FieldsConsumer Java Examples

protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      recordBlockEncodingCall();
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, 
                                                  BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new OrdsBlockTreeTermsWriter(state, 
                                                      postingsWriter,
                                                      minTermBlockSize, 
                                                      maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  minTermBlockSize, 
                                                  maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new IDVersionPostingsWriter(state.liveDocs);
  boolean success = false;
  try {
    FieldsConsumer ret = new VersionBlockTreeTermsWriter(state, 
                                                         postingsWriter,
                                                         minTermsInBlock, 
                                                         maxTermsInBlock);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsFormat delegatePostingsFormat = delegatePostingsFormat();
  if (delegatePostingsFormat == null) {
    throw new UnsupportedOperationException("Error - " + getClass().getName()
        + " has been constructed without a choice of PostingsFormat");
  }
  return new CompletionFieldsConsumer(getName(), delegatePostingsFormat, state);
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from PostingsFormat.fieldsConsumer()");
  }  
  return new CrankyFieldsConsumer(delegate.fieldsConsumer(state), random);
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNTermSelector(termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThreshold, termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException {
  final int id = nextID.getAndIncrement();

  // TODO -- ok to do this up front instead of
  // on close....?  should be ok?
  // Write our ID:
  final String idFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, ID_EXTENSION);
  IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST);
    out.writeVInt(id);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    } else {
      IOUtils.close(out);
    }
  }
  
  final RAMPostings postings = new RAMPostings();
  final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);

  synchronized(state) {
    state.put(id, postings);
  }
  return consumer;
}
 

@Override
public void write(Fields fields, NormsProducer norms) throws IOException {
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(fields);

  // Write postings
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      // Exposes only the fields from this group:
      Fields maskedFields = new FilterFields(fields) {
        @Override
        public Iterator<String> iterator() {
          return group.fields.iterator();
        }
      };

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.write(maskedFields, norms);
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 

@Override
public void merge(MergeState mergeState, NormsProducer norms) throws IOException {
  @SuppressWarnings("unchecked") Iterable<String> indexedFieldNames = () ->
      new MergedIterator<>(true,
          Arrays.stream(mergeState.fieldsProducers).map(FieldsProducer::iterator).toArray(Iterator[]::new));
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(indexedFieldNames);

  // Merge postings
  PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.merge(pfMergeState.apply(group.fields), norms);
    }
    success = true;
  } finally {
    pfMergeState.reset();
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer termsWriter = createUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
    success = true;
    return termsWriter;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 

@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  if (delegatePostingsFormat != null) {
    return new MtasFieldsConsumer(
        delegatePostingsFormat.fieldsConsumer(state), state, getName(),
        delegatePostingsFormat.getName());
  } else {
    PostingsFormat pf = Codec.forName(delegateCodecName).postingsFormat();
    return pf.fieldsConsumer(state);
  }
}
 

/**
 * Instantiates a new mtas fields consumer.
 *
 * @param fieldsConsumer
 *          the fields consumer
 * @param state
 *          the state
 * @param name
 *          the name
 * @param delegatePostingsFormatName
 *          the delegate postings format name
 */
public MtasFieldsConsumer(FieldsConsumer fieldsConsumer,
    SegmentWriteState state, String name, String delegatePostingsFormatName) {
  this.delegateFieldsConsumer = fieldsConsumer;
  this.state = state;
  this.name = name;
  this.delegatePostingsFormatName = delegatePostingsFormatName;
  // temporary fileNames
  mtasTmpFieldFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_FIELD_EXTENSION);
  mtasTmpObjectFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_OBJECT_EXTENSION);
  mtasTmpDocsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOCS_EXTENSION);
  mtasTmpDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOC_EXTENSION);
  mtasTmpDocsChainedFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_DOCS_CHAINED_EXTENSION);
  // fileNames
  mtasObjectFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION);
  mtasTermFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION);
  mtasIndexFieldFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION);
  mtasPrefixFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION);
  mtasDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION);
  mtasIndexDocIdFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_DOC_ID_EXTENSION);
  mtasIndexObjectIdFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION);
  mtasIndexObjectPositionFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION);
  mtasIndexObjectParentFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION);
}
 

CrankyFieldsConsumer(FieldsConsumer delegate, Random random) {
  this.delegate = delegate;
  this.random = random;
}
 

@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return new FieldsWriter(state);
}
 

private void mergeTerms(SegmentWriteState segmentWriteState, NormsProducer norms) throws IOException {
  try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) {
    consumer.merge(mergeState, norms);
  }
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  throw new UnsupportedOperationException("Old formats can't be used for writing");
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return delegate.fieldsConsumer(state);
}
 

AssertingFieldsConsumer(SegmentWriteState writeState, FieldsConsumer in) {
  this.writeState = writeState;
  this.in = in;
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new AssertingFieldsConsumer(state, in.fieldsConsumer(state));
}
 

@Override
protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 

protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 

public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
    SegmentWriteState state) {
  this.delegateFieldsConsumer = fieldsConsumer;
  this.state = state;
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new SimpleTextFieldsWriter(state);
}
 

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return PostingsFormat.forName("Lucene84").fieldsConsumer(state);
}