org.apache.lucene.index.CheckIndex Java Examples

private String createResultsMessage(CheckIndex.Status status) {
  String msg;
  if (status == null) {
    msg = "?";
  } else if (status.clean) {
    msg = "OK";
  } else if (status.toolOutOfDate) {
    msg = "ERROR: Can't check - tool out-of-date";
  } else {
    StringBuilder sb = new StringBuilder("BAD:");
    if (status.missingSegments) {
      sb.append(" Missing segments.");
    }
    if (status.numBadSegments > 0) {
      sb.append(" numBadSegments=");
      sb.append(status.numBadSegments);
    }
    if (status.totLoseDocCount > 0) {
      sb.append(" totLoseDocCount=");
      sb.append(status.totLoseDocCount);
    }
    msg = sb.toString();
  }
  return msg;
}
 

/** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to
 *  look for any other corruption.  */
public static CheckIndex.Status checkIndex(Directory dir, boolean doSlowChecks, boolean failFast, ByteArrayOutputStream output) throws IOException {
  if (output == null) {
    output = new ByteArrayOutputStream(1024);
  }
  // TODO: actually use the dir's locking, unless test uses a special method?
  // some tests e.g. exception tests become much more complicated if they have to close the writer
  try (CheckIndex checker = new CheckIndex(dir, NoLockFactory.INSTANCE.obtainLock(dir, "bogus"))) {
    checker.setDoSlowChecks(doSlowChecks);
    checker.setFailFast(failFast);
    checker.setInfoStream(new PrintStream(output, false, IOUtils.UTF_8), false);
    CheckIndex.Status indexStatus = checker.checkIndex(null);
    
    if (indexStatus == null || indexStatus.clean == false) {
      System.out.println("CheckIndex failed");
      System.out.println(output.toString(IOUtils.UTF_8));
      throw new RuntimeException("CheckIndex failed");
    } else {
      if (LuceneTestCase.INFOSTREAM) {
        System.out.println(output.toString(IOUtils.UTF_8));
      }
      return indexStatus;
    }
  }
}
 

/**
 * Runs check-index on the target shard and throws an exception if it failed
 */
public void checkIndex(ShardPath targetPath) throws IOException {
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name());
    try (Directory directory = new SimpleFSDirectory(targetPath.resolveIndex());
        final CheckIndex checkIndex = new CheckIndex(directory)) {
        checkIndex.setInfoStream(out);
        CheckIndex.Status status = checkIndex.checkIndex();
        out.flush();
        if (!status.clean) {
            logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
            throw new IllegalStateException("index check failure");
        }
    }
}
 

@Override
public LeafReader wrap(LeafReader reader) {
  return new FilterLeafReader(reader) {
    BitSet seenDocIDs = new BitSet();

    @Override
    public Fields getTermVectors(int docID) throws IOException {
      // if we're invoked by ParallelLeafReader then we can't do our assertion. TODO see LUCENE-6868
      if (callStackContains(ParallelLeafReader.class) == false
          && callStackContains(CheckIndex.class) == false) {
        assertFalse("Should not request TVs for doc more than once.", seenDocIDs.get(docID));
        seenDocIDs.set(docID);
      }

      return super.getTermVectors(docID);
    }

    @Override
    public CacheHelper getCoreCacheHelper() {
      return null;
    }

    @Override
    public CacheHelper getReaderCacheHelper() {
      return null;
    }
  };
}
 

@Override
public void repairIndex(CheckIndex.Status st, PrintStream ps) {
  try {
    if (dir != null) {
      IndexUtils.tryRepairIndex(dir, st, ps);
    } else {
      throw new IllegalStateException("Directory is not set.");
    }
  } catch (Exception e) {
    throw new LukeException("Failed to repair index.", e);
  }
}
 

/**
 * Checks and returns the status of the existing index in this store.
 *
 * @param out where infoStream messages should go. See {@link CheckIndex#setInfoStream(PrintStream)}
 */
public CheckIndex.Status checkIndex(PrintStream out) throws IOException {
    metadataLock.writeLock().lock();
    try (CheckIndex checkIndex = new CheckIndex(directory)) {
        checkIndex.setInfoStream(out);
        return checkIndex.checkIndex();
    } finally {
        metadataLock.writeLock().unlock();
    }
}
 

private void doCheckIndex() throws IOException {
    long timeNS = System.nanoTime();
    if (!Lucene.indexExists(store.directory())) {
        return;
    }
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name());

    if ("checksum".equalsIgnoreCase(checkIndexOnStartup)) {
        // physical verification only: verify all checksums for the latest commit
        IOException corrupt = null;
        MetadataSnapshot metadata = store.getMetadata();
        for (Map.Entry<String, StoreFileMetaData> entry : metadata.asMap().entrySet()) {
            try {
                Store.checkIntegrity(entry.getValue(), store.directory());
                out.println("checksum passed: " + entry.getKey());
            } catch (IOException exc) {
                out.println("checksum failed: " + entry.getKey());
                exc.printStackTrace(out);
                corrupt = exc;
            }
        }
        out.flush();
        if (corrupt != null) {
            logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
            throw corrupt;
        }
    } else {
        // full checkindex
        try (CheckIndex checkIndex = new CheckIndex(store.directory())) {
            checkIndex.setInfoStream(out);
            CheckIndex.Status status = checkIndex.checkIndex();
            out.flush();

            if (!status.clean) {
                if (state == IndexShardState.CLOSED) {
                    // ignore if closed....
                    return;
                }
                logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
                if ("fix".equalsIgnoreCase(checkIndexOnStartup)) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("fixing index, writing new segments file ...");
                    }
                    checkIndex.exorciseIndex(status);
                    if (logger.isDebugEnabled()) {
                        logger.debug("index fixed, wrote new segments file \"{}\"", status.segmentsFileName);
                    }
                } else {
                    // only throw a failure if we are not going to fix the index
                    throw new IllegalStateException("index check failure but can't fix it");
                }
            }
        }
    }

    if (logger.isDebugEnabled()) {
        logger.debug("check index [success]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
    }

    recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS)));
}
 

/** This runs the CheckIndex tool on the index in.  If any
 *  issues are hit, a RuntimeException is thrown; else,
 *  true is returned. */
public static CheckIndex.Status checkIndex(Directory dir) throws IOException {
  return checkIndex(dir, true);
}
 

public static CheckIndex.Status checkIndex(Directory dir, boolean doSlowChecks) throws IOException {
  return checkIndex(dir, doSlowChecks, false, null);
}
 

private void doCheckIndex() throws IOException {
    final long timeNS = System.nanoTime();
    if (!Lucene.indexExists(store.directory())) {
        return;
    }
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name());

    if ("checksum".equals(checkIndexOnStartup)) {
        // physical verification only: verify all checksums for the latest commit
        IOException corrupt = null;
        MetadataSnapshot metadata = snapshotStoreMetadata();
        for (Map.Entry<String, StoreFileMetaData> entry : metadata.asMap().entrySet()) {
            try {
                Store.checkIntegrity(entry.getValue(), store.directory());
                out.println("checksum passed: " + entry.getKey());
            } catch (IOException exc) {
                out.println("checksum failed: " + entry.getKey());
                exc.printStackTrace(out);
                corrupt = exc;
            }
        }
        out.flush();
        if (corrupt != null) {
            logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
            throw corrupt;
        }
    } else {
        // full checkindex
        final CheckIndex.Status status = store.checkIndex(out);
        out.flush();
        if (!status.clean) {
            if (state == IndexShardState.CLOSED) {
                // ignore if closed....
                return;
            }
            logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
            throw new IOException("index check failure");
        }
    }

    if (logger.isDebugEnabled()) {
        logger.debug("check index [success]\n{}", os.bytes().utf8ToString());
    }

    recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS)));
}
 

/**
 * Check the current index status.
 *
 * @param ps information stream
 * @return index status
 * @throws LukeException - if an internal error occurs when accessing index
 */
CheckIndex.Status checkIndex(PrintStream ps);
 

/**
 * Try to repair the corrupted index using previously returned index status.
 *
 * <p>This method must be called with the return value from {@link IndexTools#checkIndex(PrintStream)}.</p>
 *
 * @param st - index status
 * @param ps - information stream
 * @throws LukeException - if an internal error occurs when accessing index
 */
void repairIndex(CheckIndex.Status st, PrintStream ps);