Java Code Examples for org.elasticsearch.index.shard.IndexShard#shardId()

@Override
protected ShardStats shardOperation(IndexShardStatsRequest request, ShardRouting shardRouting) {
    IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
    IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
    // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
    if (indexShard.routingEntry() == null) {
        throw new ShardNotFoundException(indexShard.shardId());
    }

    if (!indexShard.state().equals(IndexShardState.STARTED)) {
        throw new ElasticsearchException(indexShard.shardId().toString() + " state is " + indexShard.state() + ", not started");
    }
    
    CommonStatsFlags flags = new CommonStatsFlags().clear();

    if (request.dl()) {
        flags.set(CommonStatsFlags.Flag.DL);
    }

    return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats());
}
 

private ShardRowContext(IndexShard indexShard,
                        @Nullable BlobShard blobShard,
                        ClusterService clusterService,
                        Supplier<Long> sizeSupplier) {
    this.indexShard = indexShard;
    this.blobShard = blobShard;
    this.clusterService = clusterService;
    this.sizeSupplier = sizeSupplier;
    ShardId shardId = indexShard.shardId();
    String indexName = shardId.getIndexName();
    this.id = shardId.getId();
    this.indexParts = new IndexParts(indexName);
    if (indexParts.isPartitioned()) {
        partitionIdent = indexParts.getPartitionIdent();
        RelationName relationName = indexParts.toRelationName();
        aliasName = relationName.indexNameOrAlias();
        templateName = PartitionName.templateName(relationName.schema(), relationName.name());
    } else {
        partitionIdent = "";
        aliasName = null;
        templateName = null;
    }
    path = indexShard.shardPath().getDataPath().toString();
    blobPath = blobShard == null ? null : blobShard.blobContainer().getBaseDirectory().toString();
}
 

private static Doc getDocument(IndexShard indexShard, String id, long version, long seqNo, long primaryTerm) {
    // when sequence versioning is used, this lookup will throw VersionConflictEngineException
    Doc doc = PKLookupOperation.lookupDoc(indexShard, id, Versions.MATCH_ANY, VersionType.INTERNAL, seqNo, primaryTerm);
    if (doc == null) {
        throw new DocumentMissingException(indexShard.shardId(), Constants.DEFAULT_MAPPING_TYPE, id);
    }
    if (doc.getSource() == null) {
        throw new DocumentSourceMissingException(indexShard.shardId(), Constants.DEFAULT_MAPPING_TYPE, id);
    }
    if (version != Versions.MATCH_ANY && version != doc.getVersion()) {
        throw new VersionConflictEngineException(
            indexShard.shardId(),
            id,
            "Requested version: " + version + " but got version: " + doc.getVersion());
    }
    return doc;
}
 

/**
 * Creates a new recovery target object that represents a recovery to the provided shard.
 *
 * @param indexShard                        local shard where we want to recover to
 * @param sourceNode                        source node of the recovery where we recover from
 * @param listener                          called when recovery is completed/failed
 * @param ensureClusterStateVersionCallback callback to ensure that the current node is at least on a cluster state with the provided
 *                                          version; necessary for primary relocation so that new primary knows about all other ongoing
 *                                          replica recoveries when replicating documents (see {@link RecoverySourceHandler})
 */
public RecoveryTarget(final IndexShard indexShard,
               final DiscoveryNode sourceNode,
               final PeerRecoveryTargetService.RecoveryListener listener,
               final LongConsumer ensureClusterStateVersionCallback) {
    super("recovery_status");
    this.cancellableThreads = new CancellableThreads();
    this.recoveryId = ID_GENERATOR.incrementAndGet();
    this.listener = listener;
    this.logger = Loggers.getLogger(getClass(), indexShard.shardId());
    this.indexShard = indexShard;
    this.sourceNode = sourceNode;
    this.shardId = indexShard.shardId();
    this.tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + ".";
    this.store = indexShard.store();
    this.ensureClusterStateVersionCallback = ensureClusterStateVersionCallback;
    // make sure the store is not released until we are done.
    store.incRef();
    indexShard.recoveryStats().incCurrentAsTarget();
}
 

public static Translog.Location performOnReplica(ResyncReplicationRequest request, IndexShard replica) throws Exception {
    Translog.Location location = null;
    /*
     * Operations received from resync do not have auto_id_timestamp individually, we need to bootstrap this max_seen_timestamp
     * (at least the highest timestamp from any of these operations) to make sure that we will disable optimization for the same
     * append-only requests with timestamp (sources of these operations) that are replicated; otherwise we may have duplicates.
     */
    replica.updateMaxUnsafeAutoIdTimestamp(request.getMaxSeenAutoIdTimestampOnPrimary());
    for (Translog.Operation operation : request.getOperations()) {
        final Engine.Result operationResult = replica.applyTranslogOperation(operation, Engine.Operation.Origin.REPLICA);
        if (operationResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            throw new TransportReplicationAction.RetryOnReplicaException(replica.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + operationResult.getRequiredMappingUpdate());
        }
        location = syncOperationResultOrThrow(operationResult, location);
    }
    if (request.getTrimAboveSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
        replica.trimOperationOfPreviousPrimaryTerms(request.getTrimAboveSeqNo());
    }
    return location;
}
 

@Override
protected IndexItemResponse insert(ShardUpsertRequest request,
                                   ShardUpsertRequest.Item item,
                                   IndexShard indexShard,
                                   boolean isRetry,
                                   @Nullable ReturnValueGen returnGen,
                                   @Nullable InsertSourceGen insertSourceGen) throws Exception {
    throw new VersionConflictEngineException(
        indexShard.shardId(),
        item.id(),
        "document with id: " + item.id() + " already exists in '" + request.shardId().getIndexName() + '\'');
}
 

@Test
public void testDuplicateRecoveries() throws IOException {
    IndexShard primary = newStartedShard(true);
    PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(
        mock(TransportService.class), mock(IndicesService.class),
        new RecoverySettings(
            Settings.EMPTY,
            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(
        primary.shardId(),
        randomAlphaOfLength(10),
        getFakeDiscoNode("source"),
        getFakeDiscoNode("target"),
        Store.MetadataSnapshot.EMPTY,
        randomBoolean(),
        randomLong(),
        SequenceNumbers.UNASSIGNED_SEQ_NO);
    RecoverySourceHandler handler = peerRecoverySourceService.ongoingRecoveries
        .addNewRecovery(startRecoveryRequest, primary);
    DelayRecoveryException delayRecoveryException = expectThrows(
        DelayRecoveryException.class,
        () -> peerRecoverySourceService.ongoingRecoveries.addNewRecovery(
            startRecoveryRequest,
            primary));
    assertThat(delayRecoveryException.getMessage(), containsString("recovery with same target already registered"));
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    // re-adding after removing previous attempt works
    handler = peerRecoverySourceService.ongoingRecoveries.addNewRecovery(startRecoveryRequest, primary);
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    closeShards(primary);
}
 

private Stream<ShardSegment> buildShardSegment(IndexShard indexShard) {
    try {
        List<Segment> segments = indexShard.segments(false);
        ShardId shardId = indexShard.shardId();
        return segments.stream().map(
            sgmt -> new ShardSegment(shardId.getId(),
                                     shardId.getIndexName(),
                                     sgmt,
                                     indexShard.routingEntry().primary()));
    } catch (AlreadyClosedException ignored) {
        return Stream.empty();
    }
}
 

ShardCollectorProvider(ClusterService clusterService,
                       Schemas schemas,
                       NodeJobsCounter nodeJobsCounter,
                       Functions functions,
                       ThreadPool threadPool,
                       Settings settings,
                       TransportActionProvider transportActionProvider,
                       IndexShard indexShard,
                       ShardRowContext shardRowContext) {
    this.shardRowContext = shardRowContext;
    shardNormalizer = new EvaluatingNormalizer(
        functions,
        RowGranularity.SHARD,
        new ShardReferenceResolver(schemas, shardRowContext),
        null
    );
    projectorFactory = new ProjectionToProjectorVisitor(
        clusterService,
        nodeJobsCounter,
        functions,
        threadPool,
        settings,
        transportActionProvider,
        new InputFactory(functions),
        shardNormalizer,
        t -> null,
        t -> null,
        indexShard.indexSettings().getIndexVersionCreated(),
        indexShard.shardId()
    );
}
 

/**
 * Tries to acquire reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally
 * and replication of the operation to all replica shards is completed / failed (see {@link ReplicationOperation}).
 */
private void acquirePrimaryShardReference(ShardId shardId, String allocationId, long primaryTerm,
                                          ActionListener<PrimaryShardReference> onReferenceAcquired, Object debugInfo) {
    IndexShard indexShard = getIndexShard(shardId);
    // we may end up here if the cluster state used to route the primary is so stale that the underlying
    // index shard was replaced with a replica. For example - in a two node cluster, if the primary fails
    // the replica will take over and a replica will be assigned to the first node.
    if (indexShard.routingEntry().primary() == false) {
        throw new ReplicationOperation.RetryOnPrimaryException(indexShard.shardId(),
            "actual shard is not a primary " + indexShard.routingEntry());
    }
    final String actualAllocationId = indexShard.routingEntry().allocationId().getId();
    if (actualAllocationId.equals(allocationId) == false) {
        throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId);
    }
    final long actualTerm = indexShard.getPendingPrimaryTerm();
    if (actualTerm != primaryTerm) {
        throw new ShardNotFoundException(shardId, "expected aID [{}] with term [{}] but found [{}]", allocationId,
            primaryTerm, actualTerm);
    }

    ActionListener<Releasable> onAcquired = new ActionListener<Releasable>() {
        @Override
        public void onResponse(Releasable releasable) {
            onReferenceAcquired.onResponse(new PrimaryShardReference(indexShard, releasable));
        }

        @Override
        public void onFailure(Exception e) {
            onReferenceAcquired.onFailure(e);
        }
    };

    indexShard.acquirePrimaryOperationPermit(onAcquired, executor, debugInfo);
}
 

public RecoveryStatus(IndexShard indexShard, DiscoveryNode sourceNode, RecoveryTarget.RecoveryListener listener) {

        super("recovery_status");
        this.recoveryId = idGenerator.incrementAndGet();
        this.listener = listener;
        this.logger = Loggers.getLogger(getClass(), indexShard.indexSettings(), indexShard.shardId());
        this.indexShard = indexShard;
        this.sourceNode = sourceNode;
        this.shardId = indexShard.shardId();
        this.tempFilePrefix = RECOVERY_PREFIX + indexShard.recoveryState().getTimer().startTime() + ".";
        this.store = indexShard.store();
        // make sure the store is not released until we are done.
        store.incRef();
        indexShard.recoveryStats().incCurrentAsTarget();
    }
 

public CrateSearchContext createContext(
        int jobSearchContextId,
        IndexShard indexshard,
        Engine.Searcher engineSearcher,
        WhereClause whereClause) {

    ShardId shardId = indexshard.shardId();
    SearchShardTarget searchShardTarget = new SearchShardTarget(
            clusterService.state().nodes().localNodeId(),
            shardId.getIndex(),
            shardId.id()
    );
    IndexService indexService = indexshard.indexService();
    CrateSearchContext searchContext = new CrateSearchContext(
            jobSearchContextId,
            System.currentTimeMillis(),
            searchShardTarget,
            engineSearcher,
            indexService,
            indexshard,
            scriptService,
            pageCacheRecycler,
            bigArrays,
            threadPool.estimatedTimeInMillisCounter(),
            Optional.<Scroll>absent()
    );
    LuceneQueryBuilder.Context context = luceneQueryBuilder.convert(
            whereClause,  indexService.mapperService(), indexService.fieldData(), indexService.cache());
    searchContext.parsedQuery(new ParsedQuery(context.query(), EMPTY_NAMED_FILTERS));

    Float minScore = context.minScore();
    if (minScore != null) {
        searchContext.minimumScore(minScore);
    }

    return searchContext;
}
 

@Override
protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting shardRouting) {
    IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
    IndexShard indexShard = indexService.getShard(shardRouting.shardId().id());
    // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
    if (indexShard.routingEntry() == null) {
        throw new ShardNotFoundException(indexShard.shardId());
    }

    CommonStatsFlags flags = new CommonStatsFlags().clear();

    if (request.docs()) {
        flags.set(CommonStatsFlags.Flag.Docs);
    }
    if (request.store()) {
        flags.set(CommonStatsFlags.Flag.Store);
    }
    if (request.completion()) {
        flags.set(CommonStatsFlags.Flag.Completion);
        flags.completionDataFields(request.completionFields());
    }

    CommitStats commitStats;
    SeqNoStats seqNoStats;
    try {
        commitStats = indexShard.commitStats();
        seqNoStats = indexShard.seqNoStats();
    } catch (AlreadyClosedException e) {
        // shard is closed - no stats is fine
        commitStats = null;
        seqNoStats = null;
    }

    return new ShardStats(
        indexShard.routingEntry(),
        indexShard.shardPath(),
        new CommonStats(indexShard, flags),
        commitStats,
        seqNoStats
    );
}
 

@Override
protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting shardRouting) {
    IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
    IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id());
    // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet
    if (indexShard.routingEntry() == null) {
        throw new ShardNotFoundException(indexShard.shardId());
    }

    CommonStatsFlags flags = new CommonStatsFlags().clear();

    if (request.docs()) {
        flags.set(CommonStatsFlags.Flag.Docs);
    }
    if (request.store()) {
        flags.set(CommonStatsFlags.Flag.Store);
    }
    if (request.indexing()) {
        flags.set(CommonStatsFlags.Flag.Indexing);
        flags.types(request.types());
    }
    if (request.get()) {
        flags.set(CommonStatsFlags.Flag.Get);
    }
    if (request.search()) {
        flags.set(CommonStatsFlags.Flag.Search);
        flags.groups(request.groups());
    }
    if (request.merge()) {
        flags.set(CommonStatsFlags.Flag.Merge);
    }
    if (request.refresh()) {
        flags.set(CommonStatsFlags.Flag.Refresh);
    }
    if (request.flush()) {
        flags.set(CommonStatsFlags.Flag.Flush);
    }
    if (request.warmer()) {
        flags.set(CommonStatsFlags.Flag.Warmer);
    }
    if (request.queryCache()) {
        flags.set(CommonStatsFlags.Flag.QueryCache);
    }
    if (request.fieldData()) {
        flags.set(CommonStatsFlags.Flag.FieldData);
        flags.fieldDataFields(request.fieldDataFields());
    }
    if (request.percolate()) {
        flags.set(CommonStatsFlags.Flag.Percolate);
    }
    if (request.segments()) {
        flags.set(CommonStatsFlags.Flag.Segments);
    }
    if (request.completion()) {
        flags.set(CommonStatsFlags.Flag.Completion);
        flags.completionDataFields(request.completionFields());
    }
    if (request.translog()) {
        flags.set(CommonStatsFlags.Flag.Translog);
    }
    if (request.suggest()) {
        flags.set(CommonStatsFlags.Flag.Suggest);
    }
    if (request.requestCache()) {
        flags.set(CommonStatsFlags.Flag.RequestCache);
    }
    if (request.recovery()) {
        flags.set(CommonStatsFlags.Flag.Recovery);
    }
    if (request.dl()) {
        flags.set(CommonStatsFlags.Flag.DL);
    }
    if (request.reindex()) {
        flags.set(CommonStatsFlags.Flag.Reindex);
    }

    return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats());
}
 

private boolean hasPercolatorType(IndexShard indexShard) {
    ShardId otherShardId = indexShard.shardId();
    return shardId.equals(otherShardId) && mapperService.hasMapping(PercolatorService.TYPE_NAME);
}
 

@Override
protected WriteReplicaResult<ShardUpsertRequest> processRequestItemsOnReplica(IndexShard indexShard, ShardUpsertRequest request) throws IOException {
    Translog.Location location = null;
    for (ShardUpsertRequest.Item item : request.items()) {
        if (item.source() == null) {
            if (logger.isTraceEnabled()) {
                logger.trace("[{} (R)] Document with id {}, has no source, primary operation must have failed",
                    indexShard.shardId(), item.id());
            }
            continue;
        }
        SourceToParse sourceToParse = new SourceToParse(
            indexShard.shardId().getIndexName(),
            item.id(),
            item.source(),
            XContentType.JSON
        );

        Engine.IndexResult indexResult = indexShard.applyIndexOperationOnReplica(
            item.seqNo(),
            item.version(),
            Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
            false,
            sourceToParse
        );
        if (indexResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            // Even though the primary waits on all nodes to ack the mapping changes to the master
            // (see MappingUpdatedAction.updateMappingOnMaster) we still need to protect against missing mappings
            // and wait for them. The reason is concurrent requests. Request r1 which has new field f triggers a
            // mapping update. Assume that that update is first applied on the primary, and only later on the replica
            // (it’s happening concurrently). Request r2, which now arrives on the primary and which also has the new
            // field f might see the updated mapping (on the primary), and will therefore proceed to be replicated
            // to the replica. When it arrives on the replica, there’s no guarantee that the replica has already
            // applied the new mapping, so there is no other option than to wait.
            throw new TransportReplicationAction.RetryOnReplicaException(indexShard.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + indexResult.getRequiredMappingUpdate());
        }
        location = indexResult.getTranslogLocation();
    }
    return new WriteReplicaResult<>(request, location, null, indexShard, logger);
}
 

public ShardGetService(IndexShard indexShard,
                       MapperService mapperService) {
    super(indexShard.shardId(), indexShard.indexSettings());
    this.mapperService = mapperService;
    this.indexShard = indexShard;
}
 

public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
    CommonStats oldStats = new CommonStats(flags);

    if (includePrevious) {
        Flag[] setFlags = flags.getFlags();
        for (Flag flag : setFlags) {
            switch (flag) {
                case Get:
                    oldStats.get.add(oldShardsStats.getStats);
                    break;
                case Indexing:
                    oldStats.indexing.add(oldShardsStats.indexingStats);
                    break;
                case Search:
                    oldStats.search.add(oldShardsStats.searchStats);
                    break;
                case Merge:
                    oldStats.merge.add(oldShardsStats.mergeStats);
                    break;
                case Refresh:
                    oldStats.refresh.add(oldShardsStats.refreshStats);
                    break;
                case Recovery:
                    oldStats.recoveryStats.add(oldShardsStats.recoveryStats);
                    break;
                case Flush:
                    oldStats.flush.add(oldShardsStats.flushStats);
                    break;
            }
        }
    }

    Map<Index, List<IndexShardStats>> statsByShard = Maps.newHashMap();
    for (IndexServiceInjectorPair value : indices.values()) {
        IndexService indexService = value.getIndexService();
        for (IndexShard indexShard : indexService) {
            try {
                if (indexShard.routingEntry() == null) {
                    continue;
                }
                IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indexShard, flags), indexShard.commitStats()) });
                if (!statsByShard.containsKey(indexService.index())) {
                    statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                } else {
                    statsByShard.get(indexService.index()).add(indexShardStats);
                }
            } catch (IllegalIndexShardStateException e) {
                // we can safely ignore illegal state on ones that are closing for example
                logger.trace("{} ignoring shard stats", e, indexShard.shardId());
            }
        }
    }
    return new NodeIndicesStats(oldStats, statsByShard);
}
 

/**
 * Prepares an update request by converting it into an index request.
 * <p/>
 * TODO: detect a NOOP and return an update response if true
 */
@SuppressWarnings("unchecked")
private SourceAndVersion prepareUpdate(DocTableInfo tableInfo,
                                       ShardUpsertRequest request,
                                       ShardUpsertRequest.Item item,
                                       IndexShard indexShard) throws ElasticsearchException {
    final GetResult getResult = indexShard.getService().get(request.type(), item.id(),
            new String[]{RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME},
            true, Versions.MATCH_ANY, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false);

    if (!getResult.isExists()) {
        throw new DocumentMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (getResult.internalSourceRef() == null) {
        // no source, we can't do nothing, through a failure...
        throw new DocumentSourceMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (item.version() != Versions.MATCH_ANY && item.version() != getResult.getVersion()) {
        throw new VersionConflictEngineException(
                indexShard.shardId(), Constants.DEFAULT_MAPPING_TYPE, item.id(), getResult.getVersion(), item.version());
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();

    updatedSourceAsMap = sourceAndContent.v2();

    SymbolToFieldExtractorContext ctx = new SymbolToFieldExtractorContext(functions, item.insertValues());

    Map<String, Object> pathsToUpdate = new LinkedHashMap<>();
    Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>();
    for (int i = 0; i < request.updateColumns().length; i++) {
        /**
         * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint
         * the data might be returned in the wrong format (date as string instead of long)
         */
        String columnPath = request.updateColumns()[i];
        Object value = SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).apply(getResult);
        ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath));
        if (referenceInfo instanceof GeneratedReferenceInfo) {
            updatedGeneratedColumns.put(columnPath, value);

        } else {
            pathsToUpdate.put(columnPath, value);
        }
    }

    processGeneratedColumns(tableInfo, pathsToUpdate, updatedGeneratedColumns, request.validateGeneratedColumns(), getResult);

    updateSourceByPaths(updatedSourceAsMap, pathsToUpdate);

    try {
        XContentBuilder builder = XContentFactory.contentBuilder(updateSourceContentType);
        builder.map(updatedSourceAsMap);
        return new SourceAndVersion(builder.bytes(), getResult.getVersion());
    } catch (IOException e) {
        throw new ElasticsearchGenerationException("Failed to generate [" + updatedSourceAsMap + "]", e);
    }
}