org.apache.lucene.index.IndexCommit Java Examples

public static SimpleOrderedMap<Object> getIndexInfo(DirectoryReader reader) throws IOException {
  Directory dir = reader.directory();
  SimpleOrderedMap<Object> indexInfo = new SimpleOrderedMap<>();

  indexInfo.add("numDocs", reader.numDocs());
  indexInfo.add("maxDoc", reader.maxDoc());
  indexInfo.add("deletedDocs", reader.maxDoc() - reader.numDocs());
  indexInfo.add("indexHeapUsageBytes", getIndexHeapUsed(reader));

  indexInfo.add("version", reader.getVersion());  // TODO? Is this different then: IndexReader.getCurrentVersion( dir )?
  indexInfo.add("segmentCount", reader.leaves().size());
  indexInfo.add("current", closeSafe( reader::isCurrent));
  indexInfo.add("hasDeletions", reader.hasDeletions() );
  indexInfo.add("directory", dir );
  IndexCommit indexCommit = reader.getIndexCommit();
  String segmentsFileName = indexCommit.getSegmentsFileName();
  indexInfo.add("segmentsFile", segmentsFileName);
  indexInfo.add("segmentsFileSizeInBytes", getSegmentsFileLength(indexCommit));
  Map<String,String> userData = indexCommit.getUserData();
  indexInfo.add("userData", userData);
  String s = userData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
  if (s != null) {
    indexInfo.add("lastModified", new Date(Long.parseLong(s)));
  }
  return indexInfo;
}
 

/**
 * Atomically Saves (via reference counting) & Returns the specified commit if available.
 * <p>
 * If the return value is non-null, then the caller <em>MUST</em> call {@link #releaseCommitPoint} 
 * when finished using it in order to decrement the reference count, or the commit will be preserved 
 * in the Directory forever.
 * </p>
 *
 * @return the commit point with the specified generation, or null if not available
 * @see #saveCommitPoint
 * @see #releaseCommitPoint
 */
public synchronized IndexCommit getAndSaveCommitPoint(Long generation) {
  if (null == generation) {
    throw new NullPointerException("generation to get and save must not be null");
  }
  final IndexCommit commit = knownCommits.get(generation);
  if ( (null != commit && false != commit.isDeleted())
       || (null == commit && null != latestCommit && generation < latestCommit.getGeneration()) ) {
    throw new IllegalStateException
      ("Specified index generation is too old to be saved: " + generation);
  }
  final AtomicInteger refCount
    = savedCommits.computeIfAbsent(generation, s -> { return new AtomicInteger(); });
  final int currentCount = refCount.incrementAndGet();
  log.debug("Saving generation={}, refCount={}", generation, currentCount);
  return commit;
}
 

/**
 * Find the highest index position of a safe index commit whose max sequence number is not greater than the global checkpoint.
 * Index commits with different translog UUID will be filtered out as they don't belong to this engine.
 */
private static int indexOfKeptCommits(List<? extends IndexCommit> commits, long globalCheckpoint) throws IOException {
    final String expectedTranslogUUID = commits.get(commits.size() - 1).getUserData().get(Translog.TRANSLOG_UUID_KEY);

    // Commits are sorted by age (the 0th one is the oldest commit).
    for (int i = commits.size() - 1; i >= 0; i--) {
        final Map<String, String> commitUserData = commits.get(i).getUserData();
        // Ignore index commits with different translog uuid.
        if (expectedTranslogUUID.equals(commitUserData.get(Translog.TRANSLOG_UUID_KEY)) == false) {
            return i + 1;
        }
        final long maxSeqNoFromCommit = Long.parseLong(commitUserData.get(SequenceNumbers.MAX_SEQ_NO));
        if (maxSeqNoFromCommit <= globalCheckpoint) {
            return i;
        }
    }
    // If an index was created before 6.2 or recovered from remote, we might not have a safe commit.
    // In this case, we return the oldest index commit instead.
    return 0;
}
 

/**
 * Constructor with the given index directory and callback to notify when the
 * indexes were updated.
 */
public IndexReplicationHandler(Directory indexDir, Callable<Boolean> callback) throws IOException {
  this.callback = callback;
  this.indexDir = indexDir;
  currentRevisionFiles = null;
  currentVersion = null;
  if (DirectoryReader.indexExists(indexDir)) {
    final List<IndexCommit> commits = DirectoryReader.listCommits(indexDir);
    final IndexCommit commit = commits.get(commits.size() - 1);
    currentRevisionFiles = IndexRevision.revisionFiles(commit);
    currentVersion = IndexRevision.revisionVersion(commit);
    final InfoStream infoStream = InfoStream.getDefault();
    if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
      infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion
          + " currentRevisionFiles=" + currentRevisionFiles);
      infoStream.message(INFO_STREAM_COMPONENT, "constructor(): commit=" + commit);
    }
  }
}
 

@Test
public void testCommitAge() throws InterruptedException {
  assumeFalse("This test is not working on Windows (or maybe machines with only 2 CPUs)",
    Constants.WINDOWS);

  IndexDeletionPolicyWrapper delPolicy = h.getCore().getDeletionPolicy();
  addDocs();
  Map<Long, IndexCommit> commits = delPolicy.getCommits();
  IndexCommit ic = delPolicy.getLatestCommit();
  String agestr = ((SolrDeletionPolicy) (delPolicy.getWrappedDeletionPolicy())).getMaxCommitAge().replaceAll("[a-zA-Z]", "").replaceAll("-", "");
  long age = Long.parseLong(agestr);
  Thread.sleep(age);

  assertU(adoc("id", String.valueOf(6),
          "name", "name" + String.valueOf(6)));
  assertU(optimize());
  assertQ("return all docs",
          req("id:[0 TO 6]"),
          "*[count(//doc)=6]"
  );

  commits = delPolicy.getCommits();
  assertTrue(!commits.containsKey(ic.getGeneration()));
}
 

@Override
public List<File> getFiles(long commitGen) throws LukeException {
  IndexCommit ic = getCommitMap().get(commitGen);

  if (ic == null) {
    String msg = String.format(Locale.ENGLISH, "Commit generation %d not exists.", commitGen);
    log.warn(msg);
    return Collections.emptyList();
  }

  try {
    return ic.getFileNames().stream()
        .map(name -> File.of(indexPath, name))
        .sorted(Comparator.comparing(File::getFileName))
        .collect(Collectors.toList());
  } catch (IOException e) {
    throw new LukeException(String.format(Locale.ENGLISH, "Failed to load files for commit generation %d", commitGen), e);
  }
}
 

/**
 * Retrieves the list of tlog files associated to a commit point.
 * NOTE: The commit <b>MUST</b> be reserved before calling this method
 */
List<Map<String, Object>> getTlogFileList(IndexCommit commit) throws IOException {
  long maxVersion = this.getMaxVersion(commit);
  CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog();
  String[] logList = ulog.getLogList(new File(ulog.getLogDir()));
  List<Map<String, Object>> tlogFiles = new ArrayList<>();
  for (String fileName : logList) {
    // filter out tlogs that are older than the current index commit generation, so that the list of tlog files is
    // in synch with the latest index commit point
    long startVersion = Math.abs(Long.parseLong(fileName.substring(fileName.lastIndexOf('.') + 1)));
    if (startVersion < maxVersion) {
      Map<String, Object> fileMeta = new HashMap<>();
      fileMeta.put(NAME, fileName);
      fileMeta.put(SIZE, new File(ulog.getLogDir(), fileName).length());
      tlogFiles.add(fileMeta);
    }
  }
  return tlogFiles;
}
 

@Override
public void onCommit(List<? extends IndexCommit> commits) throws IOException {
  _writeLock.lock();
  try {
    int size = commits.size();
    for (int i = 0; i < size - 1; i++) {
      IndexCommit indexCommit = commits.get(i);
      long generation = indexCommit.getGeneration();
      if (!_generationsToNames.containsKey(generation)) {
        indexCommit.delete();
      }
    }
  } finally {
    _writeLock.unlock();
  }
}
 

public void createSnapshot(String name, DirectoryReader reader, String context) throws IOException {
  _writeLock.lock();
  try {
    if (_namesToGenerations.containsKey(name)) {
      throw new IOException("Snapshot [" + name + "] already exists.");
    }
    LOG.info("Creating snapshot [{0}] in [{1}].", name, context);
    IndexCommit indexCommit = reader.getIndexCommit();
    long generation = indexCommit.getGeneration();
    _namesToGenerations.put(name, generation);
    Set<String> names = _generationsToNames.get(generation);
    if (names == null) {
      names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
      _generationsToNames.put(generation, names);
    }
    names.add(name);
    storeGenerations();
  } finally {
    _writeLock.unlock();
  }
}
 

/**
 * Read the last segments info from the commit pointed to by the searcher manager
 */
protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManager sm, final Store store) throws IOException {
    IndexSearcher searcher = sm.acquire();
    try {
        IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit();
        return Lucene.readSegmentInfos(latestCommit);
    } catch (IOException e) {
        // Fall back to reading from the store if reading from the commit fails
        try {
            return store. readLastCommittedSegmentsInfo();
        } catch (IOException e2) {
            e2.addSuppressed(e);
            throw e2;
        }
    } finally {
        sm.release(searcher);
    }
}
 

private List<NamedList<Object>> getCommits() {
  Map<Long, IndexCommit> commits = core.getDeletionPolicy().getCommits();
  List<NamedList<Object>> l = new ArrayList<>();

  for (IndexCommit c : commits.values()) {
    try {
      NamedList<Object> nl = new NamedList<>();
      nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c));
      nl.add(GENERATION, c.getGeneration());
      List<String> commitList = new ArrayList<>(c.getFileNames().size());
      commitList.addAll(c.getFileNames());
      Collections.sort(commitList);
      nl.add(CMD_GET_FILE_LIST, commitList);
      l.add(nl);
    } catch (IOException e) {
      log.warn("Exception while reading files for commit {}", c, e);
    }
  }
  return l;
}
 

public final String toString() {
  StringBuilder sb = new StringBuilder();
  sb.append("num=").append(commits.size());
  for (IndexCommit c : commits) {
    sb.append("\n\tcommit{");
    appendDetails(sb, c);
    sb.append("}");
  }
  // add an end brace
  return sb.toString();
}
 

/**
 * Captures the most recent commit point {@link #lastCommit} or the most recent safe commit point {@link #safeCommit}.
 * Index files of the capturing commit point won't be released until the commit reference is closed.
 *
 * @param acquiringSafeCommit captures the most recent safe commit point if true; otherwise captures the most recent commit point.
 */
synchronized IndexCommit acquireIndexCommit(boolean acquiringSafeCommit) {
    assert safeCommit != null : "Safe commit is not initialized yet";
    assert lastCommit != null : "Last commit is not initialized yet";
    final IndexCommit snapshotting = acquiringSafeCommit ? safeCommit : lastCommit;
    snapshottedCommits.addTo(snapshotting, 1); // increase refCount
    return new SnapshotIndexCommit(snapshotting);
}
 

private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException {
  String segmentsFileName = commit.getSegmentsFileName();
  SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName);
  Set<String> currentFiles = new HashSet<>(infos.files(true));
  String[] allFiles = indexDir.listAll();
  for (String file : allFiles) {
    if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) {
      log.info("Found unused file: {}", file);
      return true;
    }
  }
  return false;
}
 

private IndexCommit findIndexCommit(List<IndexCommit> listCommits, long generation) throws IOException {
  for (IndexCommit commit : listCommits) {
    if (commit.getGeneration() == generation) {
      return commit;
    }
  }
  throw new IOException("Generation [" + generation + "] not found.");
}
 

private MergeSortRowIdMatcher getMergeSortRowIdMatcher(Text rowId,
    Reducer<Text, NullWritable, Text, BooleanWritable>.Context context) throws IOException {
  BlurPartitioner blurPartitioner = new BlurPartitioner();
  int shard = blurPartitioner.getShard(rowId, _numberOfShardsInTable);
  String shardName = ShardUtil.getShardName(shard);

  Path shardPath = new Path(_tablePath, shardName);
  HdfsDirectory hdfsDirectory = new HdfsDirectory(_configuration, shardPath);
  SnapshotIndexDeletionPolicy policy = new SnapshotIndexDeletionPolicy(_configuration,
      SnapshotIndexDeletionPolicy.getGenerationsPath(shardPath));
  Long generation = policy.getGeneration(_snapshot);
  if (generation == null) {
    hdfsDirectory.close();
    throw new IOException("Snapshot [" + _snapshot + "] not found in shard [" + shardPath + "]");
  }

  BlurConfiguration bc = new BlurConfiguration();
  BlockCacheDirectoryFactoryV2 blockCacheDirectoryFactoryV2 = new BlockCacheDirectoryFactoryV2(bc,
      _totalNumberOfBytes);
  _closer.register(blockCacheDirectoryFactoryV2);
  Directory dir = blockCacheDirectoryFactoryV2.newDirectory("table", "shard", hdfsDirectory, null);
  List<IndexCommit> listCommits = DirectoryReader.listCommits(dir);
  IndexCommit indexCommit = ExistingDataIndexLookupMapper.findIndexCommit(listCommits, generation, shardPath);
  DirectoryReader reader = DirectoryReader.open(indexCommit);
  _rowIdsFromIndex.setValue(getTotalNumberOfRowIds(reader));

  Path cachePath = MergeSortRowIdMatcher.getCachePath(_cachePath, _table, shardName);
  return new MergeSortRowIdMatcher(dir, generation, _configuration, cachePath, context);
}
 

private IndexSearcher getIndexSearcher(String rowId) throws IOException {
  int shard = _blurPartitioner.getShard(rowId, _numberOfShardsInTable);
  if (_indexSearcher != null) {
    if (shard != _indexShard) {
      throw new IOException("Input data is not partitioned correctly.");
    }
    return _indexSearcher;
  } else {
    _indexShard = shard;
    Path shardPath = new Path(_tablePath, ShardUtil.getShardName(_indexShard));
    HdfsDirectory hdfsDirectory = new HdfsDirectory(_configuration, shardPath);
    SnapshotIndexDeletionPolicy policy = new SnapshotIndexDeletionPolicy(_configuration,
        SnapshotIndexDeletionPolicy.getGenerationsPath(shardPath));
    Long generation = policy.getGeneration(_snapshot);
    if (generation == null) {
      hdfsDirectory.close();
      throw new IOException("Snapshot [" + _snapshot + "] not found in shard [" + shardPath + "]");
    }

    BlurConfiguration bc = new BlurConfiguration();
    BlockCacheDirectoryFactoryV2 blockCacheDirectoryFactoryV2 = new BlockCacheDirectoryFactoryV2(bc,
        _totalNumberOfBytes);
    _closer.register(blockCacheDirectoryFactoryV2);
    Directory dir = blockCacheDirectoryFactoryV2.newDirectory("table", "shard", hdfsDirectory, null);

    List<IndexCommit> listCommits = DirectoryReader.listCommits(dir);
    IndexCommit indexCommit = findIndexCommit(listCommits, generation, shardPath);
    _reader = DirectoryReader.open(indexCommit);
    return _indexSearcher = new IndexSearcher(_reader);
  }
}
 

/**
 * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code>
 *
 * @throws IOException if the index is corrupted or the segments file is not present
 */
private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException {
    assert commit == null || commit.getDirectory() == directory;
    try {
        return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit);
    } catch (EOFException eof) {
        // TODO this should be caught by lucene - EOF is almost certainly an index corruption
        throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof);
    } catch (IOException exception) {
        throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up
    } catch (Exception ex) {
        throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "commit(" + commit + ")", ex);
    }

}
 

MetadataSnapshot(IndexCommit commit, Directory directory, Logger logger) throws IOException {
    LoadedMetadata loadedMetadata = loadMetadata(commit, directory, logger);
    metadata = loadedMetadata.fileMetadata;
    commitUserData = loadedMetadata.userData;
    numDocs = loadedMetadata.numDocs;
    assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles();
}
 

/**
 * Asserts that the max_seq_no stored in the commit's user_data is never smaller than seq_no of any document in the commit.
 */
public static void assertMaxSeqNoInCommitUserData(Engine engine) throws Exception {
    List<IndexCommit> commits = DirectoryReader.listCommits(engine.store.directory());
    for (IndexCommit commit : commits) {
        try (DirectoryReader reader = DirectoryReader.open(commit)) {
            assertThat(Long.parseLong(commit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
                greaterThanOrEqualTo(maxSeqNosInReader(reader)));
        }
    }
}
 

private List<? extends IndexCommit> removeCommitsStillInUse(List<? extends IndexCommit> commits) {
  List<IndexCommit> validForRemoval = new ArrayList<IndexCommit>();
  for (IndexCommit commit : commits) {
    if (!isStillInUse(commit)) {
      validForRemoval.add(commit);
    }
  }
  return validForRemoval;
}
 

public MergeSortRowIdMatcher(Directory directory, long generation, Configuration configuration, Path cachePath,
    Progressable progressable) throws IOException {
  List<IndexCommit> listCommits = DirectoryReader.listCommits(directory);
  _indexCommit = findIndexCommit(listCommits, generation);
  _configuration = configuration;
  _cachePath = cachePath;
  _directory = directory;
  _progressable = progressable == null ? NO_OP : progressable;
  _readers = openReaders();
}
 

/**
 * Internal use for Lucene... do not explicitly call.
 * <p>
 * This Impl passes the list of commits to the delegate Policy <em>AFTER</em> wrapping each 
 * commit in a proxy class that only proxies {@link IndexCommit#delete} if they are not already saved.
 * </p>
 */
@Override
public void onCommit(List<? extends IndexCommit> list) throws IOException {
  List<IndexCommitWrapper> wrapperList = wrap(list);
  updateLatestCommit(wrapperList);
  deletionPolicy.onCommit(wrapperList);
  updateKnownCommitPoints(wrapperList);
  cleanReserves();
}
 

/**
 * <p>A helper method that attempts to determine the file length of the the segments file for the 
 * specified IndexCommit from it's Directory.
 * </p>
 * <p>
 * If any sort of {@link IOException} occurs, this method will return "-1" and swallow the exception since 
 * this may be normal if the IndexCommit is no longer "on disk".  The specific type of the Exception will 
 * affect how severely it is logged: {@link NoSuchFileException} is considered more "acceptible" then other 
 * types of IOException which may indicate an actual problem with the Directory.
 */
private static long getSegmentsFileLength(IndexCommit commit) {
  try {
    return commit.getDirectory().fileLength(commit.getSegmentsFileName());
  } catch (NoSuchFileException okException) {
    log.debug("Unable to determine the (optional) fileSize for the current IndexReader's segments file because it is "
        + "no longer in the Directory, this can happen if there are new commits since the Reader was opened"
        , okException);
  } catch (IOException strangeException) {
    log.warn("Ignoring IOException wile attempting to determine the (optional) fileSize stat for the current IndexReader's segments file",
             strangeException);
  }
  return -1;
}
 

private static IndexWriter newIndexWriter(final IndexWriterConfig.OpenMode openMode, final Directory dir, final IndexCommit commit)
    throws IOException {
    assert openMode == IndexWriterConfig.OpenMode.APPEND || commit == null : "can't specify create flag with a commit";
    IndexWriterConfig iwc = new IndexWriterConfig(null)
        .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
        .setCommitOnClose(false)
        .setIndexCommit(commit)
        // we don't want merges to happen here - we call maybe merge on the engine
        // later once we stared it up otherwise we would need to wait for it here
        // we also don't specify a codec here and merges should use the engines for this index
        .setMergePolicy(NoMergePolicy.INSTANCE)
        .setOpenMode(openMode);
    return new IndexWriter(dir, iwc);
}
 

private static IndexCommit findIndexCommit(List<IndexCommit> listCommits, long generation, Path shardDir)
    throws IOException {
  for (IndexCommit commit : listCommits) {
    if (commit.getGeneration() == generation) {
      return commit;
    }
  }
  throw new IOException("Generation [" + generation + "] not found in shard [" + shardDir + "]");
}
 

public static String getCommitVersion(SolrCore solrCore) {
  IndexCommit commit = solrCore.getDeletionPolicy().getLatestCommit();
  try {
    String commitVersion = commit.getUserData().get(SolrIndexWriter.COMMIT_COMMAND_VERSION);
    if (commitVersion == null) return null;
    else return commitVersion;
  } catch (Exception e) {
    log.warn("Cannot get commit command version from index commit point ",e);
    return null;
  }
}
 

private void openNewSearcherAndUpdateCommitPoint() throws IOException {
  RefCounted<SolrIndexSearcher> searcher = null;
  IndexCommit commitPoint;
  // must get the latest solrCore object because the one we have might be closed because of a reload
  // todo stop keeping solrCore around
  SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName());
  try {
    @SuppressWarnings({"rawtypes"})
    Future[] waitSearcher = new Future[1];
    searcher = core.getSearcher(true, true, waitSearcher, true);
    if (waitSearcher[0] != null) {
      try {
        waitSearcher[0].get();
      } catch (InterruptedException | ExecutionException e) {
        SolrException.log(log, e);
      }
    }
    commitPoint = searcher.get().getIndexReader().getIndexCommit();
  } finally {
    if (searcher != null) {
      searcher.decref();
    }
    core.close();
  }

  // update the commit point in replication handler
  replicationHandler.indexCommitPoint = commitPoint;

}
 

@Override
public IndexCommitRef acquireLastIndexCommit(final boolean flushFirst) throws EngineException {
    // we have to flush outside of the readlock otherwise we might have a problem upgrading
    // the to a write lock when we fail the engine in this operation
    if (flushFirst) {
        logger.trace("start flush for snapshot");
        flush(false, true);
        logger.trace("finish flush for snapshot");
    }
    final IndexCommit lastCommit = combinedDeletionPolicy.acquireIndexCommit(false);
    return new Engine.IndexCommitRef(lastCommit, () -> releaseIndexCommit(lastCommit));
}
 

/**
 * Retrieves the maximum version number from an index commit.
 * NOTE: The commit <b>MUST</b> be reserved before calling this method
 */
private long getMaxVersion(IndexCommit commit) throws IOException {
  try (DirectoryReader reader = DirectoryReader.open(commit)) {
    IndexSearcher searcher = new IndexSearcher(reader);
    VersionInfo vinfo = core.getUpdateHandler().getUpdateLog().getVersionInfo();
    return Math.abs(vinfo.getMaxVersionFromIndex(searcher));
  }
}