Java Code Examples for org.apache.lucene.codecs.CodecUtil#retrieveChecksum()

CompletionFieldsProducer(String codecName, SegmentReadState state, FSTLoadMode fstLoadMode) throws IOException {
  String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
  delegateFieldsProducer = null;
  boolean success = false;

  try (ChecksumIndexInput index = state.directory.openChecksumInput(indexFile, state.context)) {
    // open up dict file containing all fsts
    String dictFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, DICT_EXTENSION);
    dictIn = state.directory.openInput(dictFile, state.context);
    CodecUtil.checkIndexHeader(dictIn, codecName, COMPLETION_CODEC_VERSION, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    // just validate the footer for the dictIn
    CodecUtil.retrieveChecksum(dictIn);

    // open up index file (fieldNumber, offset)
    CodecUtil.checkIndexHeader(index, codecName, COMPLETION_CODEC_VERSION, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    // load delegate PF
    PostingsFormat delegatePostingsFormat = PostingsFormat.forName(index.readString());
    delegateFieldsProducer = delegatePostingsFormat.fieldsProducer(state);

    // read suggest field numbers and their offsets in the terms file from index
    int numFields = index.readVInt();
    readers = new HashMap<>(numFields);
    for (int i = 0; i < numFields; i++) {
      int fieldNumber = index.readVInt();
      long offset = index.readVLong();
      long minWeight = index.readVLong();
      long maxWeight = index.readVLong();
      byte type = index.readByte();
      FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNumber);
      // we don't load the FST yet
      readers.put(fieldInfo.name, new CompletionsTermsReader(dictIn, offset, minWeight, maxWeight, type, fstLoadMode));
    }
    CodecUtil.checkFooter(index);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(delegateFieldsProducer, dictIn);
    }
  }
}
 

/**
 * Create a new CompoundFileDirectory.
 */
// TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
// this need not be a "general purpose" directory anymore (it only writes index files)
public Lucene50CompoundReader(Directory directory, SegmentInfo si, IOContext context) throws IOException {
  this.directory = directory;
  this.segmentName = si.name;
  String dataFileName = IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.DATA_EXTENSION);
  String entriesFileName = IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.ENTRIES_EXTENSION);
  this.entries = readEntries(si.getId(), directory, entriesFileName);
  boolean success = false;

  long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, "");
  for(Map.Entry<String,FileEntry> ent : entries.entrySet()) {
    expectedLength += ent.getValue().length;
  }
  expectedLength += CodecUtil.footerLength(); 

  handle = directory.openInput(dataFileName, context);
  try {
    CodecUtil.checkIndexHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
    
    // NOTE: data file is too costly to verify checksum against all the bytes on open,
    // but for now we at least verify proper structure of the checksum footer: which looks
    // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
    // such as file truncation.
    CodecUtil.retrieveChecksum(handle);

    // We also validate length, because e.g. if you strip 16 bytes off the .cfs we otherwise
    // would not detect it:
    if (handle.length() != expectedLength) {
      throw new CorruptIndexException("length should be " + expectedLength + " bytes, but is " + handle.length() + " instead", handle);
    }

    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(handle);
    }
  }
}
 

Lucene80NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
  maxDoc = state.segmentInfo.maxDoc();
  String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
  int version = -1;

  // read in the entries from the metadata file.
  try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
    Throwable priorE = null;
    try {
      version = CodecUtil.checkIndexHeader(in, metaCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
      readFields(in, state.fieldInfos);
    } catch (Throwable exception) {
      priorE = exception;
    } finally {
      CodecUtil.checkFooter(in, priorE);
    }
  }

  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
  data = state.directory.openInput(dataName, state.context);
  boolean success = false;
  try {
    final int version2 = CodecUtil.checkIndexHeader(data, dataCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    if (version != version2) {
      throw new CorruptIndexException("Format versions mismatch: meta=" + version + ",data=" + version2, data);
    }

    // NOTE: data file is too costly to verify checksum against all the bytes on open,
    // but for now we at least verify proper structure of the checksum footer: which looks
    // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
    // such as file truncation.
    CodecUtil.retrieveChecksum(data);

    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(this.data);
    }
  }
}
 

FieldsIndexReader(Directory dir, String name, String suffix, String extensionPrefix, String codecName, byte[] id) throws IOException {
  try (ChecksumIndexInput metaIn = dir.openChecksumInput(IndexFileNames.segmentFileName(name, suffix, extensionPrefix + FIELDS_META_EXTENSION_SUFFIX), IOContext.READONCE)) {
    Throwable priorE = null;
    try {
      CodecUtil.checkIndexHeader(metaIn, codecName + "Meta", VERSION_START, VERSION_CURRENT, id, suffix);
      maxDoc = metaIn.readInt();
      blockShift = metaIn.readInt();
      numChunks = metaIn.readInt();
      docsStartPointer = metaIn.readLong();
      docsMeta = DirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);
      docsEndPointer = startPointersStartPointer = metaIn.readLong();
      startPointersMeta = DirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);
      startPointersEndPointer = metaIn.readLong();
      maxPointer = metaIn.readLong();
    } finally {
      CodecUtil.checkFooter(metaIn, priorE);
    }
  }

  indexInput = dir.openInput(IndexFileNames.segmentFileName(name, suffix, extensionPrefix + FIELDS_INDEX_EXTENSION_SUFFIX), IOContext.READ);
  boolean success = false;
  try {
    CodecUtil.checkIndexHeader(indexInput, codecName + "Idx", VERSION_START, VERSION_CURRENT, id, suffix);
    CodecUtil.retrieveChecksum(indexInput);
    success = true;
  } finally {
    if (success == false) {
      indexInput.close();
    }
  }
  final RandomAccessInput docsSlice = indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);
  final RandomAccessInput startPointersSlice = indexInput.randomAccessSlice(startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);
  docs = DirectMonotonicReader.getInstance(docsMeta, docsSlice);
  startPointers = DirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);
}
 

/** Opens the specified file, reads its identifying information, including file length, full index header (includes the unique segment
 *  ID) and the full footer (includes checksum), and returns the resulting {@link FileMetaData}.
 *
 *  <p>This returns null, logging a message, if there are any problems (the file does not exist, is corrupt, truncated, etc.).</p> */
public FileMetaData readLocalFileMetaData(String fileName) throws IOException {

  Map<String,FileMetaData> cache = lastFileMetaData;
  FileMetaData result;
  if (cache != null) {
    // We may already have this file cached from the last NRT point:
    result = cache.get(fileName);
  } else {
    result = null;
  }

  if (result == null) {
    // Pull from the filesystem
    long checksum;
    long length;
    byte[] header;
    byte[] footer;
    try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
        try {
          length = in.length();
          header = CodecUtil.readIndexHeader(in);
          footer = CodecUtil.readFooter(in);
          checksum = CodecUtil.retrieveChecksum(in);
        } catch (EOFException | CorruptIndexException cie) {
          // File exists but is busted: we must copy it.  This happens when node had crashed, corrupting an un-fsync'd file.  On init we try
          // to delete such unreferenced files, but virus checker can block that, leaving this bad file.
          if (VERBOSE_FILES) {
            message("file " + fileName + ": will copy [existing file is corrupt]");
          }
          return null;
        }
        if (VERBOSE_FILES) {
          message("file " + fileName + " has length=" + bytesToString(length));
        }
      } catch (FileNotFoundException | NoSuchFileException e) {
      if (VERBOSE_FILES) {
        message("file " + fileName + ": will copy [file does not exist]");
      }
      return null;
    }

    // NOTE: checksum is redundant w/ footer, but we break it out separately because when the bits cross the wire we need direct access to
    // checksum when copying to catch bit flips:
    result = new FileMetaData(header, footer, length, checksum);
  }

  return result;
}
 

/**
 * @see #UniformSplitTermsReader(PostingsReaderBase, SegmentReadState, BlockDecoder, boolean)
 */
protected UniformSplitTermsReader(PostingsReaderBase postingsReader, SegmentReadState state, BlockDecoder blockDecoder,
                                  boolean dictionaryOnHeap, FieldMetadata.Serializer fieldMetadataReader,
                                  String codecName, int versionStart, int versionCurrent,
                                  String termsBlocksExtension, String dictionaryExtension) throws IOException {
   IndexInput dictionaryInput = null;
   IndexInput blockInput = null;
   boolean success = false;
   try {
     this.postingsReader = postingsReader;
     String segmentName = state.segmentInfo.name;
     String termsName = IndexFileNames.segmentFileName(segmentName, state.segmentSuffix, termsBlocksExtension);
     blockInput = state.directory.openInput(termsName, state.context);

     version = CodecUtil.checkIndexHeader(blockInput, codecName, versionStart,
         versionCurrent, state.segmentInfo.getId(), state.segmentSuffix);
     String indexName = IndexFileNames.segmentFileName(segmentName, state.segmentSuffix, dictionaryExtension);
     dictionaryInput = state.directory.openInput(indexName, state.context);

     CodecUtil.checkIndexHeader(dictionaryInput, codecName, version, version, state.segmentInfo.getId(), state.segmentSuffix);
     CodecUtil.checksumEntireFile(dictionaryInput);

     postingsReader.init(blockInput, state);
     CodecUtil.retrieveChecksum(blockInput);

     seekFieldsMetadata(blockInput);
     Collection<FieldMetadata> fieldMetadataCollection =
         readFieldsMetadata(blockInput, blockDecoder, state.fieldInfos, fieldMetadataReader, state.segmentInfo.maxDoc());

     fieldToTermsMap = new HashMap<>();
     this.blockInput = blockInput;
     this.dictionaryInput = dictionaryInput;

     fillFieldMap(postingsReader, state, blockDecoder, dictionaryOnHeap, dictionaryInput, blockInput, fieldMetadataCollection, state.fieldInfos);

     List<String> fieldNames = new ArrayList<>(fieldToTermsMap.keySet());
     Collections.sort(fieldNames);
     sortedFieldNames = Collections.unmodifiableList(fieldNames);

     success = true;
   } finally {
     if (!success) {
       IOUtils.closeWhileHandlingException(blockInput, dictionaryInput);
     }
   }
 }
 

/** expert: instantiates a new reader */
Lucene80DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
  String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
  this.maxDoc = state.segmentInfo.maxDoc();
  ramBytesUsed = RamUsageEstimator.shallowSizeOfInstance(getClass());

  // read in the entries from the metadata file.
  try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
    Throwable priorE = null;
    
    try {
      version = CodecUtil.checkIndexHeader(in, metaCodec,
                                      Lucene80DocValuesFormat.VERSION_START,
                                      Lucene80DocValuesFormat.VERSION_CURRENT,
                                      state.segmentInfo.getId(),
                                      state.segmentSuffix);
      readFields(in, state.fieldInfos);
    } catch (Throwable exception) {
      priorE = exception;
    } finally {
      CodecUtil.checkFooter(in, priorE);
    }
  }

  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
  this.data = state.directory.openInput(dataName, state.context);
  boolean success = false;
  try {
    final int version2 = CodecUtil.checkIndexHeader(data, dataCodec,
                                               Lucene80DocValuesFormat.VERSION_START,
                                               Lucene80DocValuesFormat.VERSION_CURRENT,
                                               state.segmentInfo.getId(),
                                               state.segmentSuffix);
    if (version != version2) {
      throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
    }

    // NOTE: data file is too costly to verify checksum against all the bytes on open,
    // but for now we at least verify proper structure of the checksum footer: which looks
    // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
    // such as file truncation.
    CodecUtil.retrieveChecksum(data);

    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(this.data);
    }
  }
}
 

/** Sole constructor. */
public Lucene84PostingsReader(SegmentReadState state) throws IOException {
  boolean success = false;
  IndexInput docIn = null;
  IndexInput posIn = null;
  IndexInput payIn = null;
  
  // NOTE: these data files are too costly to verify checksum against all the bytes on open,
  // but for now we at least verify proper structure of the checksum footer: which looks
  // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
  // such as file truncation.
  
  String docName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.DOC_EXTENSION);
  try {
    docIn = state.directory.openInput(docName, state.context);
    version = CodecUtil.checkIndexHeader(docIn, DOC_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    CodecUtil.retrieveChecksum(docIn);

    if (state.fieldInfos.hasProx()) {
      String proxName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.POS_EXTENSION);
      posIn = state.directory.openInput(proxName, state.context);
      CodecUtil.checkIndexHeader(posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
      CodecUtil.retrieveChecksum(posIn);

      if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
        String payName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.PAY_EXTENSION);
        payIn = state.directory.openInput(payName, state.context);
        CodecUtil.checkIndexHeader(payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
        CodecUtil.retrieveChecksum(payIn);
      }
    }

    this.docIn = docIn;
    this.posIn = posIn;
    this.payIn = payIn;
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
    }
  }
}
 

/** Sole constructor */
public Lucene86PointsReader(SegmentReadState readState) throws IOException {
  this.readState = readState;

  String metaFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
      readState.segmentSuffix,
      Lucene86PointsFormat.META_EXTENSION);
  String indexFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
      readState.segmentSuffix,
      Lucene86PointsFormat.INDEX_EXTENSION);
  String dataFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name,
      readState.segmentSuffix,
      Lucene86PointsFormat.DATA_EXTENSION);

  boolean success = false;
  try {
    indexIn = readState.directory.openInput(indexFileName, readState.context);
    CodecUtil.checkIndexHeader(indexIn,
        Lucene86PointsFormat.INDEX_CODEC_NAME,
        Lucene86PointsFormat.VERSION_START,
        Lucene86PointsFormat.VERSION_CURRENT,
        readState.segmentInfo.getId(),
        readState.segmentSuffix);

    dataIn = readState.directory.openInput(dataFileName, readState.context);
    CodecUtil.checkIndexHeader(dataIn,
        Lucene86PointsFormat.DATA_CODEC_NAME,
        Lucene86PointsFormat.VERSION_START,
        Lucene86PointsFormat.VERSION_CURRENT,
        readState.segmentInfo.getId(),
        readState.segmentSuffix);

    long indexLength = -1, dataLength = -1;
    try (ChecksumIndexInput metaIn = readState.directory.openChecksumInput(metaFileName, readState.context)) {
      Throwable priorE = null;
      try {
        CodecUtil.checkIndexHeader(metaIn,
            Lucene86PointsFormat.META_CODEC_NAME,
            Lucene86PointsFormat.VERSION_START,
            Lucene86PointsFormat.VERSION_CURRENT,
            readState.segmentInfo.getId(),
            readState.segmentSuffix);

        while (true) {
          int fieldNumber = metaIn.readInt();
          if (fieldNumber == -1) {
            break;
          } else if (fieldNumber < 0) {
            throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn);
          }
          BKDReader reader = new BKDReader(metaIn, indexIn, dataIn);
          readers.put(fieldNumber, reader);
        }
        indexLength = metaIn.readLong();
        dataLength = metaIn.readLong();
      } catch (Throwable t) {
        priorE = t;
      } finally {
        CodecUtil.checkFooter(metaIn, priorE);
      }
    }
    // At this point, checksums of the meta file have been validated so we
    // know that indexLength and dataLength are very likely correct.
    CodecUtil.retrieveChecksum(indexIn, indexLength);
    CodecUtil.retrieveChecksum(dataIn, dataLength);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(this);
    }
  }

}
 

/** Sole constructor. */
public Lucene50PostingsReader(SegmentReadState state) throws IOException {
  boolean success = false;
  IndexInput docIn = null;
  IndexInput posIn = null;
  IndexInput payIn = null;
  
  // NOTE: these data files are too costly to verify checksum against all the bytes on open,
  // but for now we at least verify proper structure of the checksum footer: which looks
  // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
  // such as file truncation.
  
  String docName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);
  try {
    docIn = state.directory.openInput(docName, state.context);
    version = CodecUtil.checkIndexHeader(docIn, DOC_CODEC, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    forUtil = new ForUtil(docIn);
    CodecUtil.retrieveChecksum(docIn);

    if (state.fieldInfos.hasProx()) {
      String proxName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);
      posIn = state.directory.openInput(proxName, state.context);
      CodecUtil.checkIndexHeader(posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
      CodecUtil.retrieveChecksum(posIn);

      if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
        String payName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.PAY_EXTENSION);
        payIn = state.directory.openInput(payName, state.context);
        CodecUtil.checkIndexHeader(payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
        CodecUtil.retrieveChecksum(payIn);
      }
    }

    this.docIn = docIn;
    this.posIn = posIn;
    this.payIn = payIn;
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
    }
  }
}
 

/**
 * Corrupts a random file at a random position
 */
public static void corruptFile(Random random, Path... files) throws IOException {
    assertTrue("files must be non-empty", files.length > 0);
    final Path fileToCorrupt = RandomPicks.randomFrom(random, files);
    assertTrue(fileToCorrupt + " is not a file", Files.isRegularFile(fileToCorrupt));
    try (Directory dir = FSDirectory.open(fileToCorrupt.toAbsolutePath().getParent())) {
        long checksumBeforeCorruption;
        try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
        }
        try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
            long maxPosition = raf.size();

            if (fileToCorrupt.getFileName().toString().endsWith(".cfs") && maxPosition > 4) {
                // TODO: it is known that Lucene does not check the checksum of CFS file (CompoundFileS, like an archive)
                // see note at https://github.com/elastic/elasticsearch/pull/33911
                // so far, don't corrupt crc32 part of checksum (last 4 bytes) of cfs file
                // checksum is 8 bytes: first 4 bytes have to be zeros, while crc32 value is not verified
                maxPosition -= 4;
            }
            final int position = random.nextInt((int) Math.min(Integer.MAX_VALUE, maxPosition));
            corruptAt(fileToCorrupt, raf, position);
        }

        long checksumAfterCorruption;
        long actualChecksumAfterCorruption;
        try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            assertThat(input.getFilePointer(), is(0L));
            input.seek(input.length() - 8); // one long is the checksum... 8 bytes
            checksumAfterCorruption = input.getChecksum();
            actualChecksumAfterCorruption = input.readLong();
        }
        // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions
        // in the checksum which is ok though....
        StringBuilder msg = new StringBuilder();
        msg.append("before: [").append(checksumBeforeCorruption).append("] ");
        msg.append("after: [").append(checksumAfterCorruption).append("] ");
        msg.append("checksum value after corruption: ").append(actualChecksumAfterCorruption).append("] ");
        msg.append("file: ").append(fileToCorrupt.getFileName()).append(" length: ");
        msg.append(dir.fileLength(fileToCorrupt.getFileName().toString()));
        logger.info("Checksum {}", msg);
        assumeTrue("Checksum collision - " + msg.toString(),
                checksumAfterCorruption != checksumBeforeCorruption // collision
                        || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted
        assertThat("no file corrupted", fileToCorrupt, notNullValue());
    }
}