Java Code Examples for org.elasticsearch.index.translog.Translog#Snapshot

SnapshotSender(Logger logger, SyncAction syncAction, ResyncTask task, ShardId shardId, String primaryAllocationId, long primaryTerm,
               Translog.Snapshot snapshot, int chunkSizeInBytes, long startingSeqNo, long maxSeqNo,
               long maxSeenAutoIdTimestamp, ActionListener<Void> listener) {
    this.logger = logger;
    this.syncAction = syncAction;
    this.task = task;
    this.shardId = shardId;
    this.primaryAllocationId = primaryAllocationId;
    this.primaryTerm = primaryTerm;
    this.snapshot = snapshot;
    this.chunkSizeInBytes = chunkSizeInBytes;
    this.startingSeqNo = startingSeqNo;
    this.maxSeqNo = maxSeqNo;
    this.maxSeenAutoIdTimestamp = maxSeenAutoIdTimestamp;
    this.listener = listener;
    task.setTotalOperations(snapshot.totalOperations());
}
 

@Override
public boolean hasCompleteOperationHistory(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        return getMinRetainedSeqNo() <= startingSeqNo;
    } else {
        final long currentLocalCheckpoint = getLocalCheckpointTracker().getProcessedCheckpoint();
        final LocalCheckpointTracker tracker = new LocalCheckpointTracker(startingSeqNo, startingSeqNo - 1);
        try (Translog.Snapshot snapshot = getTranslog().newSnapshotFromMinSeqNo(startingSeqNo)) {
            Translog.Operation operation;
            while ((operation = snapshot.next()) != null) {
                if (operation.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    tracker.markSeqNoAsProcessed(operation.seqNo());
                }
            }
        }
        return tracker.getProcessedCheckpoint() >= currentLocalCheckpoint;
    }
}
 

@Override
public Translog.Snapshot newChangesSnapshot(String source, MapperService mapperService,
                                            long fromSeqNo, long toSeqNo, boolean requiredFullRange) throws IOException {
    if (softDeleteEnabled == false) {
        throw new IllegalStateException("accessing changes snapshot requires soft-deletes enabled");
    }
    ensureOpen();
    refreshIfNeeded(source, toSeqNo);
    Searcher searcher = acquireSearcher(source, SearcherScope.INTERNAL);
    try {
        LuceneChangesSnapshot snapshot = new LuceneChangesSnapshot(
            searcher, mapperService, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE, fromSeqNo, toSeqNo, requiredFullRange);
        searcher = null;
        return snapshot;
    } catch (Exception e) {
        try {
            maybeFailEngine("acquire changes snapshot", e);
        } catch (Exception inner) {
            e.addSuppressed(inner);
        }
        throw e;
    } finally {
        IOUtils.close(searcher);
    }
}
 

@Override
public Translog.Snapshot newChangesSnapshot(String source, MapperService mapperService,
                                            long fromSeqNo, long toSeqNo, boolean requiredFullRange) throws IOException {
    // TODO: Should we defer the refresh until we really need it?
    ensureOpen();
    refreshIfNeeded(source, toSeqNo);
    Searcher searcher = acquireSearcher(source, SearcherScope.INTERNAL);
    try {
        LuceneChangesSnapshot snapshot = new LuceneChangesSnapshot(
            searcher, mapperService, LuceneChangesSnapshot.DEFAULT_BATCH_SIZE, fromSeqNo, toSeqNo, requiredFullRange);
        searcher = null;
        return snapshot;
    } catch (Exception e) {
        try {
            maybeFailEngine("acquire changes snapshot", e);
        } catch (Exception inner) {
            e.addSuppressed(inner);
        }
        throw e;
    } finally {
        IOUtils.close(searcher);
    }
}
 

/**
 * Perform phase2 of the recovery process
 * <p/>
 * Phase2 takes a snapshot of the current translog *without* acquiring the
 * write lock (however, the translog snapshot is a point-in-time view of
 * the translog). It then sends each translog operation to the target node
 * so it can be replayed into the new shard.
 */
public void phase2(Translog.Snapshot snapshot) {
    if (shard.state() == IndexShardState.CLOSED) {
        throw new IndexShardClosedException(request.shardId());
    }
    cancellableThreads.checkForCancel();

    StopWatch stopWatch = new StopWatch().start();

    logger.trace("{} recovery [phase2] to {}: sending transaction log operations", request.shardId(), request.targetNode());
    // Send all the snapshot's translog operations to the target
    int totalOperations = sendSnapshot(snapshot);
    stopWatch.stop();
    logger.trace("{} recovery [phase2] to {}: took [{}]", request.shardId(), request.targetNode(), stopWatch.totalTime());
    response.phase2Time = stopWatch.totalTime().millis();
    response.phase2Operations = totalOperations;
}
 

/**
 * Perform phase2 of the recovery process
 * <p>
 * Phase2 takes a snapshot of the current translog *without* acquiring the
 * write lock (however, the translog snapshot is a point-in-time view of
 * the translog). It then sends each translog operation to the target node
 * so it can be replayed into the new shard.
 */
public void phase2(Translog.Snapshot snapshot) {
    if (shard.state() == IndexShardState.CLOSED) {
        throw new IndexShardClosedException(request.shardId());
    }
    cancellableThreads.checkForCancel();

    StopWatch stopWatch = new StopWatch().start();

    logger.trace("{} recovery [phase2] to {}: sending transaction log operations", request.shardId(), request.targetNode());
    // Send all the snapshot's translog operations to the target
    int totalOperations = sendSnapshot(snapshot);
    stopWatch.stop();
    logger.trace("{} recovery [phase2] to {}: took [{}]", request.shardId(), request.targetNode(), stopWatch.totalTime());
    response.phase2Time = stopWatch.totalTime().millis();
    response.phase2Operations = totalOperations;
}
 

protected void recoverFromTranslog(EngineConfig engineConfig, Translog.TranslogGeneration translogGeneration) throws IOException {
    int opsRecovered = 0;
    final TranslogRecoveryPerformer handler = engineConfig.getTranslogRecoveryPerformer();
    try (Translog.Snapshot snapshot = translog.newSnapshot()) {
        opsRecovered = handler.recoveryFromSnapshot(this, snapshot);
    } catch (Throwable e) {
        throw new EngineException(shardId, "failed to recover from translog", e);
    }

    // flush if we recovered something or if we have references to older translogs
    // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0 length.
    if (opsRecovered > 0) {
        logger.trace("flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]",
                opsRecovered, translogGeneration == null ? null : translogGeneration.translogFileGeneration, translog
                        .currentFileGeneration());
        flush(true, true);
    } else if (translog.isCurrent(translogGeneration) == false) {
        commitIndexWriter(indexWriter, translog, lastCommittedSegmentInfos.getUserData().get(Engine.SYNC_COMMIT_ID));
    }
}
 

/**
 * Replays translog operations from the provided translog {@code snapshot} to the current engine using the given {@code origin}.
 * The callback {@code onOperationRecovered} is notified after each translog operation is replayed successfully.
 */
int runTranslogRecovery(Engine engine, Translog.Snapshot snapshot, Engine.Operation.Origin origin,
                        Runnable onOperationRecovered) throws IOException {
    int opsRecovered = 0;
    Translog.Operation operation;
    while ((operation = snapshot.next()) != null) {
        try {
            logger.trace("[translog] recover op {}", operation);
            Engine.Result result = applyTranslogOperation(engine, operation, origin);
            switch (result.getResultType()) {
                case FAILURE:
                    throw result.getFailure();
                case MAPPING_UPDATE_REQUIRED:
                    throw new IllegalArgumentException("unexpected mapping update: " + result.getRequiredMappingUpdate());
                case SUCCESS:
                    break;
                default:
                    throw new AssertionError("Unknown result type [" + result.getResultType() + "]");
            }

            opsRecovered++;
            onOperationRecovered.run();
        } catch (Exception e) {
            if (ExceptionsHelper.status(e) == RestStatus.BAD_REQUEST) {
                // mainly for MapperParsingException and Failure to detect xcontent
                logger.info("ignoring recovery of a corrupt translog entry", e);
            } else {
                throw ExceptionsHelper.convertToRuntime(e);
            }
        }
    }
    return opsRecovered;
}
 

/**
 * Reads all engine operations that have been processed by the engine from Lucene index.
 * The returned operations are sorted and de-duplicated, thus each sequence number will be have at most one operation.
 */
public static List<Translog.Operation> readAllOperationsInLucene(Engine engine, MapperService mapper) throws IOException {
    final List<Translog.Operation> operations = new ArrayList<>();
    long maxSeqNo = Math.max(0, ((InternalEngine) engine).getLocalCheckpointTracker().getMaxSeqNo());
    try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapper, 0, maxSeqNo, false)) {
        Translog.Operation op;
        while ((op = snapshot.next()) != null) {
            operations.add(op);
        }
    }
    return operations;
}
 

/**
 * Creates a new history snapshot for reading operations since the provided seqno.
 * The returned snapshot can be retrieved from either Lucene index or translog files.
 */
@Override
public Translog.Snapshot readHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        return newChangesSnapshot(source, mapperService, Math.max(0, startingSeqNo), Long.MAX_VALUE, false);
    } else {
        return getTranslog().newSnapshotFromMinSeqNo(startingSeqNo);
    }
}
 

private Translog.Snapshot newTranslogSnapshot(List<Translog.Operation> operations,
                                              List<Translog.Operation> operationsToSkip) {
    return new Translog.Snapshot() {
        int index = 0;
        int skippedCount = 0;

        @Override
        public int totalOperations() {
            return operations.size();
        }

        @Override
        public int skippedOperations() {
            return skippedCount;
        }

        @Override
        public Translog.Operation next() {
            while (index < operations.size()) {
                Translog.Operation op = operations.get(index++);
                if (operationsToSkip.contains(op)) {
                    skippedCount++;
                } else {
                    return op;
                }
            }
            return null;
        }

        @Override
        public void close() {

        }
    };
}
 

/**
 * Returns the estimated number of history operations whose seq# at least the provided seq# in this engine.
 */
@Override
public int estimateNumberOfHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        try (Translog.Snapshot snapshot = newChangesSnapshot(source, mapperService, Math.max(0, startingSeqNo),
            Long.MAX_VALUE, false)) {
            return snapshot.totalOperations();
        }
    } else {
        return getTranslog().estimateTotalOperationsFromMinSeq(startingSeqNo);
    }
}
 

@Override
public int restoreLocalHistoryFromTranslog(TranslogRecoveryRunner translogRecoveryRunner) throws IOException {
    try (ReleasableLock ignored = readLock.acquire()) {
        ensureOpen();
        final long localCheckpoint = localCheckpointTracker.getCheckpoint();
        try (Translog.Snapshot snapshot = getTranslog().newSnapshotFromMinSeqNo(localCheckpoint + 1)) {
            return translogRecoveryRunner.run(this, snapshot);
        }
    }
}
 

/**
 * Perform phase two of the recovery process.
 * <p>
 * Phase two uses a snapshot of the current translog *without* acquiring the write lock (however, the translog snapshot is
 * point-in-time view of the translog). It then sends each translog operation to the target node so it can be replayed into the new
 * shard.
 *
 * @param startingSeqNo              the sequence number to start recovery from, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if all
 *                                   ops should be sent
 * @param requiredSeqNoRangeStart    the lower sequence number of the required range (ending with endingSeqNo)
 * @param endingSeqNo                the highest sequence number that should be sent
 * @param snapshot                   a snapshot of the translog
 * @param maxSeenAutoIdTimestamp     the max auto_id_timestamp of append-only requests on the primary
 * @param maxSeqNoOfUpdatesOrDeletes the max seq_no of updates or deletes on the primary after these operations were executed on it.
 * @param listener                   a listener which will be notified with the local checkpoint on the target.
 */
void phase2(long startingSeqNo,
            long requiredSeqNoRangeStart,
            long endingSeqNo,
            Translog.Snapshot snapshot,
            long maxSeenAutoIdTimestamp,
            long maxSeqNoOfUpdatesOrDeletes,
            ActionListener<SendSnapshotResult> listener) throws IOException {
    assert requiredSeqNoRangeStart <= endingSeqNo + 1
        : "requiredSeqNoRangeStart " + requiredSeqNoRangeStart + " is larger than endingSeqNo " + endingSeqNo;
    assert startingSeqNo <= requiredSeqNoRangeStart
        : "startingSeqNo " + startingSeqNo + " is larger than requiredSeqNoRangeStart " + requiredSeqNoRangeStart;
    if (shard.state() == IndexShardState.CLOSED) {
        throw new IndexShardClosedException(request.shardId());
    }
    logger.trace("recovery [phase2]: sending transaction log operations (seq# from [" + startingSeqNo + "], "
        + "required [" + requiredSeqNoRangeStart + ":" + endingSeqNo + "]");

    final AtomicInteger skippedOps = new AtomicInteger();
    final AtomicInteger totalSentOps = new AtomicInteger();
    final LocalCheckpointTracker requiredOpsTracker = new LocalCheckpointTracker(endingSeqNo, requiredSeqNoRangeStart - 1);
    final AtomicInteger lastBatchCount = new AtomicInteger(); // used to estimate the count of the subsequent batch.
    final CheckedSupplier<List<Translog.Operation>, IOException> readNextBatch = () -> {
        // We need to synchronized Snapshot#next() because it's called by different threads through sendBatch.
        // Even though those calls are not concurrent, Snapshot#next() uses non-synchronized state and is not multi-thread-compatible.
        synchronized (snapshot) {
            final List<Translog.Operation> ops = lastBatchCount.get() > 0 ? new ArrayList<>(lastBatchCount.get()) : new ArrayList<>();
            long batchSizeInBytes = 0L;
            Translog.Operation operation;
            while ((operation = snapshot.next()) != null) {
                if (shard.state() == IndexShardState.CLOSED) {
                    throw new IndexShardClosedException(request.shardId());
                }
                cancellableThreads.checkForCancel();
                final long seqNo = operation.seqNo();
                if (seqNo < startingSeqNo || seqNo > endingSeqNo) {
                    skippedOps.incrementAndGet();
                    continue;
                }
                ops.add(operation);
                batchSizeInBytes += operation.estimateSize();
                totalSentOps.incrementAndGet();
                requiredOpsTracker.markSeqNoAsProcessed(seqNo);

                // check if this request is past bytes threshold, and if so, send it off
                if (batchSizeInBytes >= chunkSizeInBytes) {
                    break;
                }
            }
            lastBatchCount.set(ops.size());
            return ops;
        }
    };

    final StopWatch stopWatch = new StopWatch().start();
    final ActionListener<Long> batchedListener = ActionListener.wrap(
        targetLocalCheckpoint -> {
            assert snapshot.totalOperations() == snapshot.skippedOperations() + skippedOps.get() + totalSentOps.get()
                : String.format(Locale.ROOT, "expected total [%d], overridden [%d], skipped [%d], total sent [%d]",
                                snapshot.totalOperations(), snapshot.skippedOperations(), skippedOps.get(), totalSentOps.get());
            if (requiredOpsTracker.getProcessedCheckpoint() < endingSeqNo) {
                throw new IllegalStateException("translog replay failed to cover required sequence numbers" +
                                                " (required range [" + requiredSeqNoRangeStart + ":" + endingSeqNo + "). first missing op is ["
                                                + (requiredOpsTracker.getProcessedCheckpoint() + 1) + "]");
            }
            stopWatch.stop();
            final TimeValue tookTime = stopWatch.totalTime();
            logger.trace("recovery [phase2]: took [{}]", tookTime);
            listener.onResponse(new SendSnapshotResult(targetLocalCheckpoint, totalSentOps.get(), tookTime));
        },
        listener::onFailure
    );

    sendBatch(readNextBatch, true, SequenceNumbers.UNASSIGNED_SEQ_NO, snapshot.totalOperations(),
              maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, batchedListener);
}
 

@Override
protected int sendSnapshot(Translog.Snapshot snapshot) {
    logger.trace("{} skipping recovery of translog snapshot on shared filesystem to: {}",
            shard.shardId(), request.targetNode());
    return 0;
}
 

/**
 * Creates a new history snapshot from Lucene for reading operations whose seqno in the requesting seqno range (both inclusive)
 */
public abstract Translog.Snapshot newChangesSnapshot(String source, MapperService mapperService,
                                                     long fromSeqNo, long toSeqNo, boolean requiredFullRange) throws IOException;
 

/**
 * Creates a new history snapshot for reading operations since {@code startingSeqNo} (inclusive).
 * The returned snapshot can be retrieved from either Lucene index or translog files.
 */
public abstract Translog.Snapshot readHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException;
 

/**
 * Creates a new changes snapshot for reading operations whose seq_no are between {@code fromSeqNo}(inclusive)
 * and {@code toSeqNo}(inclusive). The caller has to close the returned snapshot after finishing the reading.
 *
 * @param source            the source of the request
 * @param fromSeqNo         the from seq_no (inclusive) to read
 * @param toSeqNo           the to seq_no (inclusive) to read
 * @param requiredFullRange if {@code true} then {@link Translog.Snapshot#next()} will throw {@link IllegalStateException}
 *                          if any operation between {@code fromSeqNo} and {@code toSeqNo} is missing.
 *                          This parameter should be only enabled when the entire requesting range is below the global checkpoint.
 */
public Translog.Snapshot newChangesSnapshot(String source, long fromSeqNo, long toSeqNo, boolean requiredFullRange) throws IOException {
    return getEngine().newChangesSnapshot(source, mapperService, fromSeqNo, toSeqNo, requiredFullRange);
}
 

/**
 * Creates a new history snapshot for reading operations since the provided starting seqno (inclusive).
 * The returned snapshot can be retrieved from either Lucene index or translog files.
 */
public Translog.Snapshot getHistoryOperations(String source, long startingSeqNo) throws IOException {
    return getEngine().readHistoryOperations(source, mapperService, startingSeqNo);
}
 

int run(Engine engine, Translog.Snapshot snapshot) throws IOException;