org.apache.lucene.index.IndexFormatTooOldException Java Examples

/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        byte[] bytes = ByteStreams.toByteArray(inputStream);
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.
 */
public final T read(Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
             // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            indexInput.readLong(); // version currently unused
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice, contentSize))) {
                    return fromXContent(parser);
                }
            }
        } catch(CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

/** Like {@link
 *  #checkHeader(DataInput,String,int,int)} except this
 *  version assumes the first int has already been read
 *  and validated from the input. */
public static int checkHeaderNoMagic(DataInput in, String codec, int minVersion, int maxVersion) throws IOException {
  final String actualCodec = in.readString();
  if (!actualCodec.equals(codec)) {
    throw new CorruptIndexException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec, in);
  }

  final int actualVersion = in.readInt();
  if (actualVersion < minVersion) {
    throw new IndexFormatTooOldException(in, actualVersion, minVersion, maxVersion);
  }
  if (actualVersion > maxVersion) {
    throw new IndexFormatTooNewException(in, actualVersion, minVersion, maxVersion);
  }

  return actualVersion;
}
 

/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    final BytesReference bytes = Streams.readFully(blobContainer.readBlob(blobName));
    final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
    try (ByteArrayIndexInput indexInput =
             new ByteArrayIndexInput(resourceDesc, BytesReference.toBytes(bytes))) {
        CodecUtil.checksumEntireFile(indexInput);
        CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
        long filePointer = indexInput.getFilePointer();
        long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
        try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
                                                                 bytes.slice((int) filePointer, (int) contentSize), XContentType.SMILE)) {
            return reader.apply(parser);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
        // we trick this into a dedicated exception with the original stacktrace
        throw new CorruptStateException(ex);
    }
}
 

private void failStoreIfCorrupted(Throwable t) {
    if (t instanceof CorruptIndexException || t instanceof IndexFormatTooOldException || t instanceof IndexFormatTooNewException) {
        try {
            store.markStoreCorrupted((IOException) t);
        } catch (IOException e) {
            logger.warn("store cannot be marked as corrupted", e);
        }
    }
}
 

/**
 * Restores a file
 *
 * @param fileInfo file to be restored
 */
private void restoreFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo, final Store store) throws IOException {
    boolean success = false;

    try (InputStream stream = fileInputStream(fileInfo)) {
        try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
            final byte[] buffer = new byte[bufferSize];
            int length;
            while ((length = stream.read(buffer)) > 0) {
                indexOutput.writeBytes(buffer, 0, length);
                recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.physicalName(), length);
            }
            Store.verify(indexOutput);
            indexOutput.close();
            store.directory().sync(Collections.singleton(fileInfo.physicalName()));
            success = true;
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            try {
                store.markStoreCorrupted(ex);
            } catch (IOException e) {
                LOGGER.warn("store cannot be marked as corrupted", e);
            }
            throw ex;
        } finally {
            if (success == false) {
                store.deleteQuiet(fileInfo.physicalName());
            }
        }
    }
}
 

/**
 * Returns the last committed segments info for this store
 *
 * @throws IOException if the index is corrupted or the segments file is not present
 */
public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
    failIfCorrupted();
    try {
        return readSegmentsInfo(null, directory());
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
        markStoreCorrupted(ex);
        throw ex;
    }
}
 

/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.
 */
public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            if (xContentType != FORMAT) {
                throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType);
            }
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (InputStreamIndexInput in = new InputStreamIndexInput(slice, contentSize)) {
                    try (XContentParser parser = XContentFactory.xContent(FORMAT)
                            .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
                                in)) {
                        return fromXContent(parser);
                    }
                }
            }
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

public static IOException unwrapCorruption(Throwable t) {
    return (IOException) unwrap(t, CorruptIndexException.class, 
                                   IndexFormatTooOldException.class, 
                                   IndexFormatTooNewException.class);
}
 

/**
 * Restores a file
 * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
 * added to the {@code failures} list
 *
 * @param fileInfo file to be restored
 */
private void restoreFile(final FileInfo fileInfo) throws IOException {
    boolean success = false;

    try (InputStream partSliceStream = new PartSliceStream(blobContainer, fileInfo)) {
        final InputStream stream;
        if (restoreRateLimiter == null) {
            stream = partSliceStream;
        } else {
            stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreThrottleListener);
        }
        try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
            final byte[] buffer = new byte[BUFFER_SIZE];
            int length;
            while ((length = stream.read(buffer)) > 0) {
                indexOutput.writeBytes(buffer, 0, length);
                recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.name(), length);
            }
            Store.verify(indexOutput);
            indexOutput.close();
            // write the checksum
            if (fileInfo.metadata().hasLegacyChecksum()) {
                Store.LegacyChecksums legacyChecksums = new Store.LegacyChecksums();
                legacyChecksums.add(fileInfo.metadata());
                legacyChecksums.write(store);

            }
            store.directory().sync(Collections.singleton(fileInfo.physicalName()));
            success = true;
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            try {
                store.markStoreCorrupted(ex);
            } catch (IOException e) {
                logger.warn("store cannot be marked as corrupted", e);
            }
            throw ex;
        } finally {
            if (success == false) {
                store.deleteQuiet(fileInfo.physicalName());
            }
        }
    }
}
 

/**
 * Instantiates a new mtas fields producer.
 *
 * @param state the state
 * @param name the name
 * @throws IOException Signals that an I/O exception has occurred.
 */
public MtasFieldsProducer(SegmentReadState state, String name)
    throws IOException {
  String postingsFormatName = null;
  indexInputList = new HashMap<>();
  indexInputOffsetList = new HashMap<>();
  version = MtasCodecPostingsFormat.VERSION_CURRENT;    
  postingsFormatName = addIndexInputToList("object", openMtasFile(state, name,
      MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION), postingsFormatName);
  addIndexInputToList("term",
      openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION),
      postingsFormatName);
  addIndexInputToList("prefix", openMtasFile(state, name,
      MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION), postingsFormatName);
  addIndexInputToList("field",
      openMtasFile(state, name, MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION),
      postingsFormatName);
  addIndexInputToList("indexDocId",
      openMtasFile(state, name,
          MtasCodecPostingsFormat.MTAS_INDEX_DOC_ID_EXTENSION),
      postingsFormatName);
  addIndexInputToList("indexObjectId",
      openMtasFile(state, name,
          MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION),
      postingsFormatName);
  try {
    addIndexInputToList(
        "doc", openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_DOC_EXTENSION, version, version),
        postingsFormatName);
    addIndexInputToList("indexObjectPosition",
        openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION,
            version, version),
        postingsFormatName);
    addIndexInputToList("indexObjectParent",
        openMtasFile(state, name,
            MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION,
            version, version),
        postingsFormatName);
  } catch (IndexFormatTooOldException e) {
    log.debug(e);
    throw new IOException(
        "This MTAS doesn't support your index version, please upgrade");
  }
  // Load the delegate postingsFormatName from this file
  this.delegateFieldsProducer = PostingsFormat.forName(postingsFormatName)
      .fieldsProducer(state);      
}
 

public static IOException unwrapCorruption(Throwable t) {
    return (IOException) unwrap(t, CorruptIndexException.class,
                                   IndexFormatTooOldException.class,
                                   IndexFormatTooNewException.class);
}