Java Code Examples for org.elasticsearch.index.translog.Translog#Operation

@Override
public Translog.Operation next() throws IOException {
    Translog.Operation op = null;
    for (int idx = nextDocIndex(); idx != -1; idx = nextDocIndex()) {
        op = readDocAsOp(idx);
        if (op != null) {
            break;
        }
    }
    if (requiredFullRange) {
        rangeCheck(op);
    }
    if (op != null) {
        lastSeenSeqNo = op.seqNo();
    }
    return op;
}
 

/**
 * 
 * @param operation
 * @return
 * @throws IOException 
 */
public Tuple<Future<DLSN>, Tuple<BytesReference, Long>> writeOperation(Translog.Operation operation, AtomicLong txid) throws IOException {
    BytesStreamOutput out = new BytesStreamOutput();
    try (ReleasableLock lock = writeLock.acquire()) {
        Future<DLSN> writeResult = null;
        out.writeByte(operation.opType().id());
        operation.writeTo(out);
        BytesReference bytes = out.bytes();
        LogRecord logRecord = new LogRecord(txid.incrementAndGet(), bytes.toBytes());
        writeResult = logWriter.write(logRecord);
        sizeInBytes += (20 + logRecord.getPayload().length);
        ++ numOperations;
        return new Tuple<Future<DLSN>, Tuple<BytesReference, Long>>(writeResult, new Tuple<BytesReference, Long>(bytes, txid.get()));
    } catch (TransactionIdOutOfOrderException e) {
        throw e;
    } finally {
        out.close();
    }
}
 

@Override
public void indexTranslogOperations(List<Translog.Operation> operations,
                                    int totalTranslogOps,
                                    long maxSeenAutoIdTimestampOnPrimary,
                                    long maxSeqNoOfDeletesOrUpdatesOnPrimary,
                                    ActionListener<Long> listener) {
    final RecoveryTranslogOperationsRequest request = new RecoveryTranslogOperationsRequest(
        recoveryId,
        shardId,
        operations,
        totalTranslogOps,
        maxSeenAutoIdTimestampOnPrimary,
        maxSeqNoOfDeletesOrUpdatesOnPrimary);
    transportService.submitRequest(
        targetNode,
        PeerRecoveryTargetService.Actions.TRANSLOG_OPS,
        request,
        translogOpsRequestOptions,
        new ActionListenerResponseHandler<>(
            ActionListener.wrap(
                r -> listener.onResponse(r.localCheckpoint), listener::onFailure),
            RecoveryTranslogOperationsResponse::new,
            ThreadPool.Names.GENERIC)
    );
}
 

public int recoveryFromSnapshot(Engine engine, Translog.Snapshot snapshot) throws IOException {
    Translog.Operation operation;
    int opsRecovered = 0;
    while ((operation = snapshot.next()) != null) {
        try {
            performRecoveryOperation(engine, operation, true);
            opsRecovered++;
        } catch (ElasticsearchException e) {
            if (e.status() == RestStatus.BAD_REQUEST) {
                // mainly for MapperParsingException and Failure to detect xcontent
                logger.info("ignoring recovery of a corrupt translog entry", e);
            } else {
                throw e;
            }
        }
    }
    return opsRecovered;
}
 

RecoveryTranslogOperationsRequest(long recoveryId,
                                  ShardId shardId,
                                  List<Translog.Operation> operations,
                                  int totalTranslogOps,
                                  long maxSeenAutoIdTimestampOnPrimary,
                                  long maxSeqNoOfUpdatesOrDeletesOnPrimary) {
    this.recoveryId = recoveryId;
    this.shardId = shardId;
    this.operations = operations;
    this.totalTranslogOps = totalTranslogOps;
    this.maxSeenAutoIdTimestampOnPrimary = maxSeenAutoIdTimestampOnPrimary;
    this.maxSeqNoOfUpdatesOrDeletesOnPrimary = maxSeqNoOfUpdatesOrDeletesOnPrimary;
}
 

/**
 * this method is useless now, since we will skip translog recovery from remote node
 */
public int performBatchRecovery(Iterable<Translog.Operation> operations) {
    if (state != IndexShardState.RECOVERING) {
        throw new IndexShardNotRecoveringException(shardId, state);
    }

    // This will activate our shard so we get our fair share of the indexing buffer during recovery:
    markLastWrite();
    return 0;
}
 

/**
 * Reads all engine operations that have been processed by the engine from Lucene index.
 * The returned operations are sorted and de-duplicated, thus each sequence number will be have at most one operation.
 */
public static List<Translog.Operation> readAllOperationsInLucene(Engine engine, MapperService mapper) throws IOException {
    final List<Translog.Operation> operations = new ArrayList<>();
    long maxSeqNo = Math.max(0, ((InternalEngine) engine).getLocalCheckpointTracker().getMaxSeqNo());
    try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapper, 0, maxSeqNo, false)) {
        Translog.Operation op;
        while ((op = snapshot.next()) != null) {
            operations.add(op);
        }
    }
    return operations;
}
 

/**
 * Returns a new empty translog operation for the given {@link Translog.Operation.Type}
 */
public static Translog.Operation newOperationFromType(Translog.Operation.Type type) throws IOException {
    switch (type) {
        case CREATE:
            return new Translog.Create();
        case DELETE:
            return new Translog.Delete();
        case SAVE:
            return new Translog.Index();
        default:
            throw new IOException("No type for [" + type + "]");
    }
}
 

private void sendBatch(CheckedSupplier<List<Translog.Operation>, IOException> nextBatch,
                       boolean firstBatch,
                       long targetLocalCheckpoint,
                       int totalTranslogOps,
                       long maxSeenAutoIdTimestamp,
                       long maxSeqNoOfUpdatesOrDeletes,
                       ActionListener<Long> listener) throws IOException {
    final List<Translog.Operation> operations = nextBatch.get();
    // send the leftover operations or if no operations were sent, request
    // the target to respond with its local checkpoint
    if (operations.isEmpty() == false || firstBatch) {
        cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(
            operations, totalTranslogOps, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes,
            ActionListener.wrap(newCheckpoint ->
                    sendBatch(
                        nextBatch,
                        false,
                        SequenceNumbers.max(targetLocalCheckpoint, newCheckpoint),
                        totalTranslogOps,
                        maxSeenAutoIdTimestamp,
                        maxSeqNoOfUpdatesOrDeletes,
                        listener),
                listener::onFailure
            ))
        );
    } else {
        listener.onResponse(targetLocalCheckpoint);
    }
}
 

/**
 * Applies all operations in the iterable to the current engine and returns the number of operations applied.
 * This operation will stop applying operations once an operation failed to apply.
 * Note: This method is typically used in peer recovery to replay remote transaction log entries.
 */
public int performBatchRecovery(Iterable<Translog.Operation> operations) {
    if (state != IndexShardState.RECOVERING) {
        throw new IndexShardNotRecoveringException(shardId, state);
    }

    // This will activate our shard so we get our fair share of the indexing buffer during recovery:
    markLastWrite();

    return engineConfig.getTranslogRecoveryPerformer().performBatchRecovery(engine(), operations);
}
 

@Override
public int run(Engine engine, Translog.Snapshot snapshot) throws IOException {
    int opsRecovered = 0;
    Translog.Operation operation;
    while ((operation = snapshot.next()) != null) {
        applyOperation(engine, convertToEngineOp(operation, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY));
        opsRecovered++;
    }
    engine.syncTranslog();
    return opsRecovered;
}
 

private void rangeCheck(Translog.Operation op) {
    if (op == null) {
        if (lastSeenSeqNo < toSeqNo) {
            throw new IllegalStateException("Not all operations between from_seqno [" + fromSeqNo + "] " +
                "and to_seqno [" + toSeqNo + "] found; prematurely terminated last_seen_seqno [" + lastSeenSeqNo + "]");
        }
    } else {
        final long expectedSeqNo = lastSeenSeqNo + 1;
        if (op.seqNo() != expectedSeqNo) {
            throw new IllegalStateException("Not all operations between from_seqno [" + fromSeqNo + "] " +
                "and to_seqno [" + toSeqNo + "] found; expected seqno [" + expectedSeqNo + "]; found [" + op + "]");
        }
    }
}
 

private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation operation,
                                             Engine.Operation.Origin origin) throws IOException {
    // If a translog op is replayed on the primary (eg. ccr), we need to use external instead of null for its version type.
    final VersionType versionType = (origin == Engine.Operation.Origin.PRIMARY) ? VersionType.EXTERNAL : null;
    final Engine.Result result;
    switch (operation.opType()) {
        case INDEX:
            final Translog.Index index = (Translog.Index) operation;
            // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
            // autoGeneratedID docs that are coming from the primary are updated correctly.
            result = applyIndexOperation(
                engine,
                index.seqNo(),
                index.primaryTerm(),
                index.version(),
                versionType,
                UNASSIGNED_SEQ_NO,
                0,
                index.getAutoGeneratedIdTimestamp(),
                true,
                origin,
                new SourceToParse(
                    shardId.getIndexName(),
                    index.id(),
                    index.source(),
                    XContentHelper.xContentType(index.source()), index.routing())
            );
            break;
        case DELETE:
            final Translog.Delete delete = (Translog.Delete) operation;
            result = applyDeleteOperation(engine, delete.seqNo(), delete.primaryTerm(), delete.version(), delete.type(), delete.id(),
                                          versionType, UNASSIGNED_SEQ_NO, 0, origin);
            break;
        case NO_OP:
            final Translog.NoOp noOp = (Translog.NoOp) operation;
            result = markSeqNoAsNoop(engine, noOp.seqNo(), noOp.primaryTerm(), noOp.reason(), origin);
            break;
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
    return result;
}
 

private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.Operation.Origin origin) {
    switch (operation.opType()) {
        case INDEX:
            final Translog.Index index = (Translog.Index) operation;
            final String indexName = mapperService.index().getName();
            return IndexShard.prepareIndex(
                docMapper(index.type()),
                new SourceToParse(
                    indexName,
                    index.id(),
                    index.source(),
                    XContentHelper.xContentType(index.source()),
                    index.routing()
                ),
                index.seqNo(),
                index.primaryTerm(),
                index.version(),
                null,
                origin,
                index.getAutoGeneratedIdTimestamp(),
                true,
                SequenceNumbers.UNASSIGNED_SEQ_NO,
                0
            );
        case DELETE:
            final Translog.Delete delete = (Translog.Delete) operation;
            return new Engine.Delete(delete.type(),
                                     delete.id(),
                                     delete.uid(),
                                     delete.seqNo(),
                                     delete.primaryTerm(),
                                     delete.version(),
                                     null,
                                     origin,
                                     System.nanoTime(),
                                     SequenceNumbers.UNASSIGNED_SEQ_NO,
                                     0);
        case NO_OP:
            final Translog.NoOp noOp = (Translog.NoOp) operation;
            return new Engine.NoOp(noOp.seqNo(), noOp.primaryTerm(), origin, System.nanoTime(), noOp.reason());
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
}
 

RecoveryTranslogOperationsRequest(long recoveryId, ShardId shardId, List<Translog.Operation> operations, int totalTranslogOps) {
    this.recoveryId = recoveryId;
    this.shardId = shardId;
    this.operations = operations;
    this.totalTranslogOps = totalTranslogOps;
}
 

public Engine.Result applyTranslogOperation(Translog.Operation operation, Engine.Operation.Origin origin) throws IOException {
    return applyTranslogOperation(getEngine(), operation, origin);
}
 

public Translog.Operation readFromLocal(Location location) {
    if (location == null) {
        return null;
    }
    return this.localTranslog.readFromLocal(location);
}
 

private Translog.Operation readDocAsOp(int docIndex) throws IOException {
    final LeafReaderContext leaf = parallelArray.leafReaderContexts[docIndex];
    final int segmentDocID = scoreDocs[docIndex].doc - leaf.docBase;
    final long primaryTerm = parallelArray.primaryTerm[docIndex];
    // We don't have to read the nested child documents - those docs don't have primary terms.
    if (primaryTerm == -1) {
        skippedOperations++;
        return null;
    }
    final long seqNo = parallelArray.seqNo[docIndex];
    // Only pick the first seen seq#
    if (seqNo == lastSeenSeqNo) {
        skippedOperations++;
        return null;
    }
    final long version = parallelArray.version[docIndex];
    final String sourceField = parallelArray.hasRecoverySource[docIndex] ? SourceFieldMapper.RECOVERY_SOURCE_NAME :
        SourceFieldMapper.NAME;
    final FieldsVisitor fields = new FieldsVisitor(true, sourceField);
    leaf.reader().document(segmentDocID, fields);
    fields.postProcess(mapperService);

    final Translog.Operation op;
    final boolean isTombstone = parallelArray.isTombStone[docIndex];
    if (isTombstone && fields.uid() == null) {
        op = new Translog.NoOp(seqNo, primaryTerm, fields.source().utf8ToString());
        assert version == 1L : "Noop tombstone should have version 1L; actual version [" + version + "]";
        assert assertDocSoftDeleted(leaf.reader(), segmentDocID) : "Noop but soft_deletes field is not set [" + op + "]";
    } else {
        final String id = fields.uid().id();
        final String type = fields.uid().type();
        final Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id));
        if (isTombstone) {
            op = new Translog.Delete(type, id, uid, seqNo, primaryTerm, version);
            assert assertDocSoftDeleted(leaf.reader(), segmentDocID) : "Delete op but soft_deletes field is not set [" + op + "]";
        } else {
            final BytesReference source = fields.source();
            if (source == null) {
                // TODO: Callers should ask for the range that source should be retained. Thus we should always
                // check for the existence source once we make peer-recovery to send ops after the local checkpoint.
                if (requiredFullRange) {
                    throw new IllegalStateException("source not found for seqno=" + seqNo +
                        " from_seqno=" + fromSeqNo + " to_seqno=" + toSeqNo);
                } else {
                    skippedOperations++;
                    return null;
                }
            }
            // TODO: pass the latest timestamp from engine.
            final long autoGeneratedIdTimestamp = -1;
            op = new Translog.Index(type, id, seqNo, primaryTerm, version,
                source.toBytesRef().bytes, fields.routing(), autoGeneratedIdTimestamp);
        }
    }
    assert fromSeqNo <= op.seqNo() && op.seqNo() <= toSeqNo && lastSeenSeqNo < op.seqNo() : "Unexpected operation; " +
        "last_seen_seqno [" + lastSeenSeqNo + "], from_seqno [" + fromSeqNo + "], to_seqno [" + toSeqNo + "], op [" + op + "]";
    return op;
}
 

public Translog.Operation[] getOperations() {
    return operations;
}
 

/**
 * Index a set of translog operations on the target
 *
 * @param operations                          operations to index
 * @param totalTranslogOps                    current number of total operations expected to be indexed
 * @param maxSeenAutoIdTimestampOnPrimary     the maximum auto_id_timestamp of all append-only requests processed by the primary shard
 * @param maxSeqNoOfUpdatesOrDeletesOnPrimary the max seq_no of update operations (index operations overwrite Lucene) or delete ops on
 *                                            the primary shard when capturing these operations. This value is at least as high as the
 *                                            max_seq_no_of_updates on the primary was when any of these ops were processed on it.
 * @param listener                            a listener which will be notified with the local checkpoint on the target
 *                                            after these operations are successfully indexed on the target.
 * @return the local checkpoint on the target shard
 */
void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long maxSeenAutoIdTimestampOnPrimary,
                             long maxSeqNoOfUpdatesOrDeletesOnPrimary, ActionListener<Long> listener);