Java Code Examples for org.apache.lucene.codecs.CodecUtil#writeFooter()

@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (meta != null) {
      meta.writeInt(-1); // write EOF marker
      CodecUtil.writeFooter(meta); // write checksum
    }
    if (data != null) {
      CodecUtil.writeFooter(data); // write checksum
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(data, meta);
    } else {
      IOUtils.closeWhileHandlingException(data, meta);
    }
    meta = data = null;
  }
}
 

public void testFixedLengthLiesLiesLies() throws Exception {
  // Make sure OfflineSorter catches me if I lie about the fixed value length:
  Directory dir = newDirectory();
  IndexOutput out = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
  try (ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(out)) {
    byte[] bytes = new byte[Integer.BYTES];
    random().nextBytes(bytes);
    w.write(bytes);
    CodecUtil.writeFooter(out);
  }

  OfflineSorter sorter = new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(4), OfflineSorter.MAX_TEMPFILES, Long.BYTES, null, 0);
  IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
    sorter.sort(out.getName());
    });
  assertEquals("value length is 4 but is supposed to always be 8", e.getMessage());
  dir.close();
}
 

@Override
public void close() throws IOException {
  if (out != null) {
    try {
      final long dirStart = out.getFilePointer();
      
      out.writeVInt(fields.size());
      for(FieldMetaData field : fields) {
        out.writeVInt(field.fieldInfo.number);
        out.writeVLong(field.numTerms);
        out.writeVLong(field.termsStartPointer);
        if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
          out.writeVLong(field.sumTotalTermFreq);
        }
        out.writeVLong(field.sumDocFreq);
        out.writeVInt(field.docCount);
      }
      writeTrailer(dirStart);
      CodecUtil.writeFooter(out);
    } finally {
      IOUtils.close(out, postingsWriter, termsIndexWriter);
      out = null;
    }
  }
}
 

@Override
public void write(Fields fields, NormsProducer normsProducer) throws IOException {
  BlockWriter blockWriter = new BlockWriter(blockOutput, targetNumBlockLines, deltaNumLines, blockEncoder);
  ByteBuffersDataOutput fieldsOutput = new ByteBuffersDataOutput();
  int fieldsNumber = 0;
  for (String field : fields) {
    Terms terms = fields.terms(field);
    if (terms != null) {
      TermsEnum termsEnum = terms.iterator();
      FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
      fieldsNumber += writeFieldTerms(blockWriter, fieldsOutput, termsEnum, fieldInfo, normsProducer);
    }
  }
  writeFieldsMetadata(fieldsNumber, fieldsOutput);
  CodecUtil.writeFooter(dictionaryOutput);
}
 

@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (meta != null) {
      meta.writeInt(-1); // write EOF marker
      CodecUtil.writeFooter(meta); // write checksum
    }
    if (data != null) {
      CodecUtil.writeFooter(data); // write checksum
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(data, meta);
    } else {
      IOUtils.closeWhileHandlingException(data, meta);
    }
    meta = data = null;
  }
}
 

public void testLargeCFS() throws IOException {   
  final String testfile = "_123.test";
  IOContext context = new IOContext(new FlushInfo(0, 512*1024*1024));

  Directory dir = new NRTCachingDirectory(newFSDirectory(createTempDir()), 2.0, 25.0);

  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, context)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    byte[] bytes = new byte[512];
    for(int i=0;i<1024*1024;i++) {
      out.writeBytes(bytes, 0, bytes.length);
    }
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, context);

  dir.close();
}
 

/**
 * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception
 * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>.
 */
public void markStoreCorrupted(IOException exception) throws IOException {
    ensureOpen();
    if (!isMarkedCorrupted()) {
        String uuid = CORRUPTED + UUIDs.randomBase64UUID();
        try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(output, CODEC, VERSION);
            BytesStreamOutput out = new BytesStreamOutput();
            out.writeException(exception);
            BytesReference bytes = out.bytes();
            output.writeVInt(bytes.length());
            BytesRef ref = bytes.toBytesRef();
            output.writeBytes(ref.bytes, ref.offset, ref.length);
            CodecUtil.writeFooter(output);
        } catch (IOException ex) {
            logger.warn("Can't mark store as corrupted", ex);
        }
        directory().sync(Collections.singleton(uuid));
    }
}
 

@Override
public void close() throws IOException {
  // TODO: add a finish() at least to PushBase? DV too...?
  boolean success = false;
  try {
    if (docOut != null) {
      CodecUtil.writeFooter(docOut);
    }
    if (posOut != null) {
      CodecUtil.writeFooter(posOut);
    }
    if (payOut != null) {
      CodecUtil.writeFooter(payOut);
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(docOut, posOut, payOut);
    } else {
      IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
    }
    docOut = posOut = payOut = null;
  }
}
 

public void testDoubleClose() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  assertEquals(1, cfs.listAll().length);
  cfs.close();
  cfs.close(); // second close should not throw exception
  dir.close();
}
 

@Override
public void close() throws IOException {
  if (closed) {
    return;
  }
  closed = true;
  String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
  boolean success = false;
  try (IndexOutput indexOut = state.directory.createOutput(indexFile, state.context)) {
    delegateFieldsConsumer.close();
    CodecUtil.writeIndexHeader(indexOut, codecName, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    /*
     * we write the delegate postings format name so we can load it
     * without getting an instance in the ctor
     */
    indexOut.writeString(delegatePostingsFormatName);
    // write # of seen fields
    indexOut.writeVInt(seenFields.size());
    // write field numbers and dictOut offsets
    for (Map.Entry<String, CompletionMetaData> seenField : seenFields.entrySet()) {
      FieldInfo fieldInfo = state.fieldInfos.fieldInfo(seenField.getKey());
      indexOut.writeVInt(fieldInfo.number);
      CompletionMetaData metaData = seenField.getValue();
      indexOut.writeVLong(metaData.filePointer);
      indexOut.writeVLong(metaData.minWeight);
      indexOut.writeVLong(metaData.maxWeight);
      indexOut.writeByte(metaData.type);
    }
    CodecUtil.writeFooter(indexOut);
    CodecUtil.writeFooter(dictOut);
    IOUtils.close(dictOut);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(dictOut, delegateFieldsConsumer);
    }
  }
}
 

private void writeTo(final T obj, final String blobName, final CheckedConsumer<BytesArray, IOException> consumer) throws IOException {
    final BytesReference bytes;
    try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
        if (compress) {
            try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) {
                write(obj, compressedStreamOutput);
            }
        } else {
            write(obj, bytesStreamOutput);
        }
        bytes = bytesStreamOutput.bytes();
    }
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, outputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                }
            }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        consumer.accept(new BytesArray(outputStream.toByteArray()));
    }
}
 

/** Creates a file of the specified size with random data. */
protected static void createRandomFile(Directory dir, String name, int size, byte[] segId) throws IOException {
  Random rnd = random();
  try (IndexOutput os = dir.createOutput(name, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, segId, "suffix");
    for (int i=0; i<size; i++) {
      byte b = (byte) rnd.nextInt(256);
      os.writeByte(b);
    }
    CodecUtil.writeFooter(os);
  }
}
 

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
      output.writeVInt(fi.getPointDimensionCount());
      if (fi.getPointDimensionCount() != 0) {
        output.writeVInt(fi.getPointIndexDimensionCount());
        output.writeVInt(fi.getPointNumBytes());
      }
    }
    CodecUtil.writeFooter(output);
  }
}
 

private void writeStateToFirstLocation(final T state, Path stateLocation, Directory stateDir, String tmpFileName)
        throws WriteStateException {
    try {
        deleteFileIfExists(stateLocation, stateDir, tmpFileName);
        try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
            out.writeInt(FORMAT.index());
            try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) {
                @Override
                public void close() {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                }
            })) {
                builder.startObject();
                toXContent(builder, state);
                builder.endObject();
            }
            CodecUtil.writeFooter(out);
        }

        stateDir.sync(Collections.singleton(tmpFileName));
    } catch (Exception e) {
        throw new WriteStateException(false, "failed to write state to the first location tmp file " +
                stateLocation.resolve(tmpFileName), e);
    }
}
 

private void closeWriter() throws IOException {
  if (writer != null) {
    CodecUtil.writeFooter(input);
    writer.close();
    writer = null;
  }
}
 

protected void writeFieldsMetadata(int fieldsNumber, ByteBuffersDataOutput fieldsOutput) throws IOException {
  long fieldsStartPosition = blockOutput.getFilePointer();
  blockOutput.writeVInt(fieldsNumber);
  if (blockEncoder == null) {
    writeUnencodedFieldsMetadata(fieldsOutput);
  } else {
    writeEncodedFieldsMetadata(fieldsOutput);
  }
  // Must be a fixed length. Read by UniformSplitTermsReader when seeking fields metadata.
  blockOutput.writeLong(fieldsStartPosition);
  CodecUtil.writeFooter(blockOutput);
}
 

@Override
public void close() throws IOException {
  if (out != null) {
    boolean success = false;
    try {
      // write field summary
      final long dirStart = out.getFilePointer();
      
      out.writeVInt(fields.size());
      for (FieldMetaData field : fields) {
        out.writeVInt(field.fieldInfo.number);
        out.writeVLong(field.numTerms);
        if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
          out.writeVLong(field.sumTotalTermFreq);
        }
        out.writeVLong(field.sumDocFreq);
        out.writeVInt(field.docCount);
        field.dict.save(out, out);
      }
      writeTrailer(out, dirStart);
      CodecUtil.writeFooter(out);
      success = true;
    } finally {
      if (success) {
        IOUtils.close(out, postingsWriter);
      } else {
        IOUtils.closeWhileHandlingException(out, postingsWriter);
      }
      out = null;
    }
  }
}
 

@Override
public void finish() throws IOException {
  if (finished) {
    throw new IllegalStateException("already finished");
  }
  finished = true;
  metaOut.writeInt(-1);
  CodecUtil.writeFooter(indexOut);
  CodecUtil.writeFooter(dataOut);
  metaOut.writeLong(indexOut.getFilePointer());
  metaOut.writeLong(dataOut.getFilePointer());
  CodecUtil.writeFooter(metaOut);
}
 

public void testManySubFiles() throws IOException {
  final MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("CFSManySubFiles"));
  
  final int FILE_COUNT = atLeast(500);
  
  List<String> files = new ArrayList<>();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    String file = "_123." + fileIdx;
    files.add(file);
    try (IndexOutput out = dir.createOutput(file, newIOContext(random()))) {
      CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
      out.writeByte((byte) fileIdx);
      CodecUtil.writeFooter(out);
    }
  }
  
  assertEquals(0, dir.getFileHandleCount());
  
  si.setFiles(files);
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  
  final IndexInput[] ins = new IndexInput[FILE_COUNT];
  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    ins[fileIdx] = cfs.openInput("_123." + fileIdx, newIOContext(random()));
    CodecUtil.checkIndexHeader(ins[fileIdx], "Foo", 0, 0, si.getId(), "suffix");
  }
  
  assertEquals(1, dir.getFileHandleCount());

  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    assertEquals((byte) fileIdx, ins[fileIdx].readByte());
  }
  
  assertEquals(1, dir.getFileHandleCount());
  
  for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
    ins[fileIdx].close();
  }
  cfs.close();
  
  dir.close();
}
 

protected void writeDictionary(int fieldsNumber, IndexDictionary.Builder dictionaryBuilder) throws IOException {
  if (fieldsNumber > 0) {
    writeDictionary(dictionaryBuilder);
  }
  CodecUtil.writeFooter(dictionaryOutput);
}