Java Code Examples for org.elasticsearch.index.seqno.SequenceNumbers#UNASSIGNED_SEQ_NO

public static Translog.Location performOnReplica(ResyncReplicationRequest request, IndexShard replica) throws Exception {
    Translog.Location location = null;
    /*
     * Operations received from resync do not have auto_id_timestamp individually, we need to bootstrap this max_seen_timestamp
     * (at least the highest timestamp from any of these operations) to make sure that we will disable optimization for the same
     * append-only requests with timestamp (sources of these operations) that are replicated; otherwise we may have duplicates.
     */
    replica.updateMaxUnsafeAutoIdTimestamp(request.getMaxSeenAutoIdTimestampOnPrimary());
    for (Translog.Operation operation : request.getOperations()) {
        final Engine.Result operationResult = replica.applyTranslogOperation(operation, Engine.Operation.Origin.REPLICA);
        if (operationResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            throw new TransportReplicationAction.RetryOnReplicaException(replica.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + operationResult.getRequiredMappingUpdate());
        }
        location = syncOperationResultOrThrow(operationResult, location);
    }
    if (request.getTrimAboveSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        replica.trimOperationOfPreviousPrimaryTerms(request.getTrimAboveSeqNo());
    }
    return location;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

protected IndexRequestBuilder createUpdateRequest(final EsAbstractEntity esEntity) {
    final IndexRequestBuilder builder =
            client.prepareIndex().setIndex(asEsIndex()).setId(esEntity.asDocMeta().id()).setSource(toSource(esEntity));
    final RequestOptionCall<IndexRequestBuilder> indexOption = esEntity.asDocMeta().indexOption();
    if (indexOption != null) {
        indexOption.callback(builder);
    }
    final Long seqNo = esEntity.asDocMeta().seqNo();
    if (seqNo != null && seqNo.longValue() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final Long primaryTerm = esEntity.asDocMeta().primaryTerm();
    if (primaryTerm != null && primaryTerm.longValue() != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }
    return builder;
}
 

@Override
protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
    final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
    final IndexRequestBuilder builder = createUpdateRequest(esEntity);

    final IndexResponse response = builder.execute().actionGet(indexTimeout);
    final long seqNo = response.getSeqNo();
    if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        esEntity.asDocMeta().seqNo(seqNo);
    }
    final long primaryTerm = response.getPrimaryTerm();
    if (primaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
        esEntity.asDocMeta().primaryTerm(primaryTerm);
    }

    return 1;
}
 

protected Engine.Index replicaIndexForDoc(ParsedDocument doc,
                                          long version,
                                          long seqNo,
                                          boolean isRetry) {
    return new Engine.Index(
        newUid(doc), doc, seqNo, primaryTerm.get(), version, null,
        Engine.Operation.Origin.REPLICA, System.nanoTime(), Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
        isRetry, SequenceNumbers.UNASSIGNED_SEQ_NO, 0);
}
 

@Override
public Translog.Operation next() throws IOException {
    for (; index >= 0; index--) {
        final TranslogSnapshot current = translogs[index];
        Translog.Operation op;
        while ((op = current.next()) != null) {
            if (op.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO || seenSeqNo.getAndSet(op.seqNo()) == false) {
                return op;
            } else {
                overriddenOperations++;
            }
        }
    }
    return null;
}
 

static Checkpoint emptyTranslogCheckpoint(final long offset, final long generation, final long globalCheckpoint,
                                          long minTranslogGeneration) {
    final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, 0, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 

@Test
public void testDuplicateRecoveries() throws IOException {
    IndexShard primary = newStartedShard(true);
    PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(
        mock(TransportService.class), mock(IndicesService.class),
        new RecoverySettings(
            Settings.EMPTY,
            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(
        primary.shardId(),
        randomAlphaOfLength(10),
        getFakeDiscoNode("source"),
        getFakeDiscoNode("target"),
        Store.MetadataSnapshot.EMPTY,
        randomBoolean(),
        randomLong(),
        SequenceNumbers.UNASSIGNED_SEQ_NO);
    RecoverySourceHandler handler = peerRecoverySourceService.ongoingRecoveries
        .addNewRecovery(startRecoveryRequest, primary);
    DelayRecoveryException delayRecoveryException = expectThrows(
        DelayRecoveryException.class,
        () -> peerRecoverySourceService.ongoingRecoveries.addNewRecovery(
            startRecoveryRequest,
            primary));
    assertThat(delayRecoveryException.getMessage(), containsString("recovery with same target already registered"));
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    // re-adding after removing previous attempt works
    handler = peerRecoverySourceService.ongoingRecoveries.addNewRecovery(startRecoveryRequest, primary);
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    closeShards(primary);
}
 

public StartRecoveryRequest getStartRecoveryRequest() throws IOException {
    Store.MetadataSnapshot metadataSnapshot = randomBoolean() ? Store.MetadataSnapshot.EMPTY :
        new Store.MetadataSnapshot(Collections.emptyMap(),
                                   Collections.singletonMap(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()), randomIntBetween(0, 100));
    return new StartRecoveryRequest(
        shardId,
        null,
        new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
        new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
        metadataSnapshot,
        randomBoolean(),
        randomNonNegativeLong(),
        randomBoolean() || metadataSnapshot.getHistoryUUID() == null ?
            SequenceNumbers.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
}
 

static Checkpoint readCheckpointV5_0_0(final DataInput in) throws IOException {
    final long offset = in.readLong();
    final int numOps = in.readInt();
    final long generation = in.readLong();
    final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final long globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
    final long minTranslogGeneration = -1;
    final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}
 

/**
 * Closes current reader and creates new one with new checkoint and same file channel
 */
TranslogReader closeIntoTrimmedReader(long aboveSeqNo, ChannelFactory channelFactory) throws IOException {
    if (closed.compareAndSet(false, true)) {
        Closeable toCloseOnFailure = channel;
        final TranslogReader newReader;
        try {
            if (aboveSeqNo < checkpoint.trimmedAboveSeqNo
                || aboveSeqNo < checkpoint.maxSeqNo && checkpoint.trimmedAboveSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO) {
                final Path checkpointFile = path.getParent().resolve(getCommitCheckpointFileName(checkpoint.generation));
                final Checkpoint newCheckpoint = new Checkpoint(checkpoint.offset, checkpoint.numOps,
                    checkpoint.generation, checkpoint.minSeqNo, checkpoint.maxSeqNo,
                    checkpoint.globalCheckpoint, checkpoint.minTranslogGeneration, aboveSeqNo);
                Checkpoint.write(channelFactory, checkpointFile, newCheckpoint, StandardOpenOption.WRITE);

                IOUtils.fsync(checkpointFile, false);
                IOUtils.fsync(checkpointFile.getParent(), true);

                newReader = new TranslogReader(newCheckpoint, channel, path, header);
            } else {
                newReader = new TranslogReader(checkpoint, channel, path, header);
            }
            toCloseOnFailure = null;
            return newReader;
        } finally {
            IOUtils.close(toCloseOnFailure);
        }
    } else {
        throw new AlreadyClosedException(toString() + " is already closed");
    }
}
 

public static DeletionStrategy skipDueToVersionConflict(
        VersionConflictEngineException e, long currentVersion, long term, boolean currentlyDeleted) {
    final long unassignedSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    final DeleteResult deleteResult = new DeleteResult(e, currentVersion, term, unassignedSeqNo, currentlyDeleted == false);
    return new DeletionStrategy(false, false, currentlyDeleted, unassignedSeqNo,
        Versions.NOT_FOUND, deleteResult);
}
 

protected Engine.Delete replicaDeleteForDoc(String id, long version, long seqNo, long startTime) {
    return new Engine.Delete(
        "default", id, newUid(id), seqNo, 1, version, null,
        Engine.Operation.Origin.REPLICA, startTime, SequenceNumbers.UNASSIGNED_SEQ_NO, 0);
}
 

private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.Operation.Origin origin) {
    switch (operation.opType()) {
        case INDEX:
            final Translog.Index index = (Translog.Index) operation;
            final String indexName = mapperService.index().getName();
            return IndexShard.prepareIndex(
                docMapper(index.type()),
                new SourceToParse(
                    indexName,
                    index.id(),
                    index.source(),
                    XContentHelper.xContentType(index.source()),
                    index.routing()
                ),
                index.seqNo(),
                index.primaryTerm(),
                index.version(),
                null,
                origin,
                index.getAutoGeneratedIdTimestamp(),
                true,
                SequenceNumbers.UNASSIGNED_SEQ_NO,
                0
            );
        case DELETE:
            final Translog.Delete delete = (Translog.Delete) operation;
            return new Engine.Delete(delete.type(),
                                     delete.id(),
                                     delete.uid(),
                                     delete.seqNo(),
                                     delete.primaryTerm(),
                                     delete.version(),
                                     null,
                                     origin,
                                     System.nanoTime(),
                                     SequenceNumbers.UNASSIGNED_SEQ_NO,
                                     0);
        case NO_OP:
            final Translog.NoOp noOp = (Translog.NoOp) operation;
            return new Engine.NoOp(noOp.seqNo(), noOp.primaryTerm(), origin, System.nanoTime(), noOp.reason());
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
}
 

/**
 * use in case of the index operation failed before getting to internal engine
 **/
public IndexResult(Exception failure, long version, long term) {
    this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO);
}
 

/**
 * use in case of the delete operation failed before getting to internal engine
 **/
public DeleteResult(Exception failure, long version, long term) {
    this(failure, version, term, SequenceNumbers.UNASSIGNED_SEQ_NO, false);
}
 

protected final GetResult getFromSearcher(Get get, BiFunction<String, SearcherScope, Searcher> searcherFactory,
                                          SearcherScope scope) throws EngineException {
    final Searcher searcher = searcherFactory.apply("get", scope);
    final DocIdAndVersion docIdAndVersion;
    try {
        docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.reader(), get.uid(), true);
    } catch (Exception e) {
        Releasables.closeWhileHandlingException(searcher);
        //TODO: A better exception goes here
        throw new EngineException(shardId, "Couldn't resolve version", e);
    }

    if (docIdAndVersion != null) {
        if (get.versionType().isVersionConflictForReads(docIdAndVersion.version, get.version())) {
            Releasables.close(searcher);
            throw new VersionConflictEngineException(
                shardId,
                get.id(),
                get.versionType().explainConflictForReads(docIdAndVersion.version, get.version())
            );
        }
        if (get.getIfSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && (
            get.getIfSeqNo() != docIdAndVersion.seqNo || get.getIfPrimaryTerm() != docIdAndVersion.primaryTerm)) {

            Releasables.close(searcher);
            throw new VersionConflictEngineException(
                shardId,
                get.id(),
                get.getIfSeqNo(),
                get.getIfPrimaryTerm(),
                docIdAndVersion.seqNo,
                docIdAndVersion.primaryTerm
            );
        }
    }

    if (docIdAndVersion != null) {
        // don't release the searcher on this path, it is the
        // responsibility of the caller to call GetResult.release
        return new GetResult(docIdAndVersion, searcher);
    } else {
        Releasables.close(searcher);
        return GetResult.NOT_EXISTS;
    }
}
 

public static DeletionStrategy skipDueToVersionConflict(
        VersionConflictEngineException e, long currentVersion, long term, boolean currentlyDeleted) {
    final long unassignedSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
    final DeleteResult deleteResult = new DeleteResult(e, currentVersion, term, unassignedSeqNo, currentlyDeleted == false);
    return new DeletionStrategy(false, false, currentlyDeleted, Versions.NOT_FOUND, deleteResult);
}
 

@Override
protected void doRun() throws Exception {
    long size = 0;
    final List<Translog.Operation> operations = new ArrayList<>();

    task.setPhase("collecting_ops");
    task.setResyncedOperations(totalSentOps.get());

    Translog.Operation operation;
    while ((operation = snapshot.next()) != null) {
        final long seqNo = operation.seqNo();
        if (startingSeqNo >= 0 &&
            (seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO || seqNo < startingSeqNo)) {
            totalSkippedOps.incrementAndGet();
            continue;
        }
        operations.add(operation);
        size += operation.estimateSize();
        totalSentOps.incrementAndGet();

        // check if this request is past bytes threshold, and if so, send it off
        if (size >= chunkSizeInBytes) {
            break;
        }
    }

    final long trimmedAboveSeqNo = firstMessage.get() ? maxSeqNo : SequenceNumbers.UNASSIGNED_SEQ_NO;
    // have to send sync request even in case of there are no operations to sync - have to sync trimmedAboveSeqNo at least
    if (!operations.isEmpty() || trimmedAboveSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        task.setPhase("sending_ops");
        ResyncReplicationRequest request =
            new ResyncReplicationRequest(shardId, trimmedAboveSeqNo, maxSeenAutoIdTimestamp, operations.toArray(EMPTY_ARRAY));
        logger.trace("{} sending batch of [{}][{}] (total sent: [{}], skipped: [{}])", shardId, operations.size(),
            new ByteSizeValue(size), totalSentOps.get(), totalSkippedOps.get());
        firstMessage.set(false);
        syncAction.sync(request, task, primaryAllocationId, primaryTerm, this);
    } else if (closed.compareAndSet(false, true)) {
        logger.trace("{} resync completed (total sent: [{}], skipped: [{}])", shardId, totalSentOps.get(), totalSkippedOps.get());
        listener.onResponse(null);
    }
}
 

boolean retryOnConflict() {
    return seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO && version == Versions.MATCH_ANY;
}