org.elasticsearch.index.shard.ShardId Java Examples

public ProjectionToProjectorVisitor(ClusterService clusterService,
                                    NodeJobsCounter nodeJobsCounter,
                                    Functions functions,
                                    ThreadPool threadPool,
                                    Settings settings,
                                    TransportActionProvider transportActionProvider,
                                    InputFactory inputFactory,
                                    EvaluatingNormalizer normalizer,
                                    Function<RelationName, SysRowUpdater<?>> sysUpdaterGetter,
                                    Function<RelationName, StaticTableDefinition<?>> staticTableDefinitionGetter,
                                    Version indexVersionCreated,
                                    @Nullable ShardId shardId) {
    this.clusterService = clusterService;
    this.nodeJobsCounter = nodeJobsCounter;
    this.functions = functions;
    this.threadPool = threadPool;
    this.settings = settings;
    this.transportActionProvider = transportActionProvider;
    this.inputFactory = inputFactory;
    this.normalizer = normalizer;
    this.sysUpdaterGetter = sysUpdaterGetter;
    this.staticTableDefinitionGetter = staticTableDefinitionGetter;
    this.indexVersionCreated = indexVersionCreated;
    this.shardId = shardId;
    this.numProcessors = EsExecutors.numberOfProcessors(settings);
}
 

@Override
public void logDocumentDeleted(ShardId shardId, Delete delete, DeleteResult result) {

    String effectiveUser = getUser();

    if(!checkComplianceFilter(Category.COMPLIANCE_DOC_WRITE, effectiveUser, getOrigin())) {
        return;
    }

    AuditMessage msg = new AuditMessage(Category.COMPLIANCE_DOC_WRITE, clusterService, getOrigin(), null);
    TransportAddress remoteAddress = getRemoteAddress();
    msg.addRemoteAddress(remoteAddress);
    msg.addEffectiveUser(effectiveUser);
    msg.addIndices(new String[]{shardId.getIndexName()});
    msg.addResolvedIndices(new String[]{shardId.getIndexName()});
    msg.addId(delete.id());
    msg.addShardId(shardId);
    msg.addComplianceDocVersion(result.getVersion());
    msg.addComplianceOperation(Operation.DELETE);
    save(msg);
}
 

@Test
public void testSearchWithScores() throws Exception {
    IndexWriter w = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthur"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections.singletonList(new ScoreCollectorExpression());
    Query query = fieldType.termsQuery(Collections.singletonList("Arthur"), null);
    LuceneOrderedDocCollector collector = collector(searcher, columnReferences, query, null, true);
    KeyIterable<ShardId, Row> result = collector.collect();

    assertThat(Iterables.size(result), is(2));

    Iterator<Row> values = result.iterator();

    assertThat(values.next().get(0), Matchers.is(1.0F));
    assertThat(values.next().get(0), Matchers.is(1.0F));
}
 

@Override
public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                  Settings indexSettings) {
    for (IndexEventListener listener : listeners) {
        try {
            listener.afterIndexShardClosed(shardId, indexShard, indexSettings);
        } catch (Exception e) {
            logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke after shard closed callback", shardId.getId()), e);
            throw e;
        }
    }
}
 

/**
 * Returns the expected shard size for the given shard or the default value provided if not enough information are available
 * to estimate the shards size.
 */
public static long getExpectedShardSize(ShardRouting shard,
                                        long defaultValue,
                                        ClusterInfo clusterInfo,
                                        MetaData metaData,
                                        RoutingTable routingTable) {
    final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
    if (indexMetaData.getResizeSourceIndex() != null && shard.active() == false &&
        shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
        // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
        // the worst case
        long targetShardSize = 0;
        final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex();
        final IndexMetaData sourceIndexMeta = metaData.index(mergeSourceIndex);
        if (sourceIndexMeta != null) {
            final Set<ShardId> shardIds = IndexMetaData.selectRecoverFromShards(shard.id(),
                sourceIndexMeta, indexMetaData.getNumberOfShards());
            for (IndexShardRoutingTable shardRoutingTable : routingTable.index(mergeSourceIndex.getName())) {
                if (shardIds.contains(shardRoutingTable.shardId())) {
                    targetShardSize += clusterInfo.getShardSize(shardRoutingTable.primaryShard(), 0);
                }
            }
        }
        return targetShardSize == 0 ? defaultValue : targetShardSize;
    } else {
        return clusterInfo.getShardSize(shard, defaultValue);
    }
}
 

/**
 * Initializes a new empty index, with an option to control if its from an API or not.
 */
private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 

@Override
public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                   Settings indexSettings) {
    if (indexShard != null) {
        ongoingRecoveries.cancel(indexShard, "shard is closed");
    }
}
 

public RecoverFilesRecoveryException(ShardId shardId, int numberOfFiles, ByteSizeValue totalFilesSize, Throwable cause) {
    super("Failed to transfer [{}] files with total size of [{}]", cause, numberOfFiles, totalFilesSize);
    Objects.requireNonNull(totalFilesSize, "totalFilesSize must not be null");
    setShard(shardId);
    this.numberOfFiles = numberOfFiles;
    this.totalFilesSize = totalFilesSize;
}
 

@Override
public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {
    if (indexShard != null) {
        Boolean remove = shardSet.remove(indexShard);
        if (remove == Boolean.TRUE) {
            Logger logger = Loggers.getLogger(getClass(), indexShard.shardId());
            MockFSDirectoryService.checkIndex(logger, indexShard.store(), indexShard.shardId());
        }
    }
}
 

/**
 * failure constructor
 */
public ShardsSyncedFlushResult(ShardId shardId, int totalShards, String failureReason) {
    this.syncId = null;
    this.failureReason = failureReason;
    this.shardResponses = ImmutableMap.of();
    this.shardId = shardId;
    this.totalShards = totalShards;
}
 

@Override
public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                                Settings indexSettings) {
    if (indexShard != null) {
        getStats.addTotals(indexShard.getStats());
        indexingStats.addTotals(indexShard.indexingStats());
        searchStats.addTotals(indexShard.searchStats());
        mergeStats.addTotals(indexShard.mergeStats());
        refreshStats.addTotals(indexShard.refreshStats());
        flushStats.addTotals(indexShard.flushStats());
        recoveryStats.addTotals(indexShard.recoveryStats());
    }
}
 

/**
 * Fails the given snapshot restore operation for the given shard
 */
public void failRestore(SnapshotId snapshotId, ShardId shardId) {
    logger.debug("[{}] failed to restore shard  [{}]", snapshotId, shardId);
    UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId,
            new ShardRestoreStatus(clusterService.state().nodes().localNodeId(), RestoreInProgress.State.FAILURE));
        transportService.sendRequest(clusterService.state().nodes().masterNode(),
                UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
}
 

@Override
protected void resolveRequest(MetaData metaData, String concreteIndex, IndexRequest request) {
    MappingMetaData mappingMd = null;
    if (metaData.hasIndex(concreteIndex)) {
        mappingMd = metaData.index(concreteIndex).mappingOrDefault(request.type());
    }
    request.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
    ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(), concreteIndex, request.type(), request.id(), request.routing());
    request.setShardId(shardId);
}
 

/**
 * Returns shard for the specified id if it exists otherwise returns <code>null</code>.
 */
default T getShardOrNull(ShardId shardId) {
    U indexRef = indexService(shardId.getIndex());
    if (indexRef != null) {
        return indexRef.getShardOrNull(shardId.id());
    }
    return null;
}
 

/**
 * Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
 */
public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService,
                    Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
                    MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer,
                    Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener,
                    TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, IndexSearcherWrappingService wrappingService, TranslogConfig translogConfig) {
    this.shardId = shardId;
    this.indexSettings = indexSettings;
    this.threadPool = threadPool;
    this.indexingService = indexingService;
    this.warmer = warmer;
    this.store = store;
    this.deletionPolicy = deletionPolicy;
    this.mergePolicy = mergePolicy;
    this.mergeSchedulerConfig = mergeSchedulerConfig;
    this.analyzer = analyzer;
    this.similarity = similarity;
    this.codecService = codecService;
    this.failedEngineListener = failedEngineListener;
    this.wrappingService = wrappingService;
    this.optimizeAutoGenerateId = indexSettings.getAsBoolean(EngineConfig.INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING, false);
    this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
    codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME);
    // We start up inactive and rely on IndexingMemoryController to give us our fair share once we start indexing:
    indexingBufferSize = IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER;
    gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis();
    versionMapSizeSetting = indexSettings.get(INDEX_VERSION_MAP_SIZE, DEFAULT_VERSION_MAP_SIZE);
    updateVersionMapSize();
    this.translogRecoveryPerformer = translogRecoveryPerformer;
    this.forceNewTranslog = indexSettings.getAsBoolean(INDEX_FORCE_NEW_TRANSLOG, false);
    this.queryCache = queryCache;
    this.queryCachingPolicy = queryCachingPolicy;
    this.translogConfig = translogConfig;
}
 

/**
 * success constructor
 */
public ShardsSyncedFlushResult(ShardId shardId, String syncId, int totalShards, Map<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> shardResponses) {
    this.failureReason = null;
    ImmutableMap.Builder<ShardRouting, SyncedFlushService.ShardSyncedFlushResponse> builder = ImmutableMap.builder();
    this.shardResponses = builder.putAll(shardResponses).build();
    this.syncId = syncId;
    this.totalShards = totalShards;
    this.shardId = shardId;
}
 

public DeleteProjector(ClusterService clusterService,
                       IndexNameExpressionResolver indexNameExpressionResolver,
                       Settings settings,
                       ShardId shardId,
                       TransportActionProvider transportActionProvider,
                       BulkRetryCoordinatorPool bulkRetryCoordinatorPool,
                       CollectExpression<Row, ?> collectUidExpression,
                       UUID jobId) {
    super(clusterService, settings, shardId, transportActionProvider, bulkRetryCoordinatorPool,
            collectUidExpression, jobId);
    this.indexNameExpressionResolver = indexNameExpressionResolver;
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    if (in.readBoolean()) {
        internalShardId = ShardId.readShardId(in);
    }
    index = in.readOptionalString();
    // no need to pass threading over the network, they are always false when coming throw a thread pool
}
 

/**
 * On the first call this will do an initial search and provide {@link #batchSize} number of rows
 * (or less if there aren't more available)
 * </p>
 * On subsequent calls it will return more rows (max {@link #batchSize} or less.
 * These rows are always the rows that come after the last row of the previously returned rows
 * <p/>
 * Basically, calling this function multiple times pages through the shard in batches.
 */
@Override
public KeyIterable<ShardId, Row> collect() {
    try {
        if (lastDoc == null) {
            return initialSearch();
        }
        return searchMore();
    } catch (Exception e) {
        Exceptions.rethrowUnchecked(e);
        return null;
    }
}
 

/**
 * Checks if any shards were processed that the new master doesn't know about
 */
private void syncShardStatsOnNewMaster(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    if (snapshotsInProgress == null) {
        return;
    }

    for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
        if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
            Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshot());
            if (localShards != null) {
                ImmutableOpenMap<ShardId, ShardSnapshotStatus> masterShards = snapshot.shards();
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> localShard : localShards.entrySet()) {
                    ShardId shardId = localShard.getKey();
                    ShardSnapshotStatus masterShard = masterShards.get(shardId);
                    if (masterShard != null && masterShard.state().completed() == false) {
                        final IndexShardSnapshotStatus.Copy indexShardSnapshotStatus = localShard.getValue().asCopy();
                        final Stage stage = indexShardSnapshotStatus.getStage();
                        // Master knows about the shard and thinks it has not completed
                        if (stage == Stage.DONE) {
                            // but we think the shard is done - we need to make new master know that the shard is done
                            LOGGER.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, " +
                                "updating status on the master", snapshot.snapshot(), shardId);
                            notifySuccessfulSnapshotShard(snapshot.snapshot(), shardId, localShard.getValue().generation());

                        } else if (stage == Stage.FAILURE) {
                            // but we think the shard failed - we need to make new master know that the shard failed
                            LOGGER.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, " +
                                "updating status on master", snapshot.snapshot(), shardId);
                            notifyFailedSnapshotShard(snapshot.snapshot(), shardId, indexShardSnapshotStatus.getFailure());
                        }
                    }
                }
            }
        }
    }
}
 

PKLookupTask(UUID jobId,
             int phaseId,
             String name,
             RamAccounting ramAccounting,
             Function<RamAccounting, MemoryManager> memoryManagerFactory,
             int ramAccountingBlockSizeInBytes,
             TransactionContext txnCtx,
             InputFactory inputFactory,
             PKLookupOperation pkLookupOperation,
             List<ColumnIdent> partitionedByColumns,
             List<Symbol> toCollect,
             Map<ShardId, List<PKAndVersion>> idsByShard,
             Collection<? extends Projection> shardProjections,
             RowConsumer consumer) {
    super(phaseId);
    this.jobId = jobId;
    this.name = name;
    this.ramAccounting = ramAccounting;
    this.txnCtx = txnCtx;
    this.pkLookupOperation = pkLookupOperation;
    this.idsByShard = idsByShard;
    this.shardProjections = shardProjections;
    this.consumer = consumer;
    this.memoryManagerFactory = memoryManagerFactory;
    this.ramAccountingBlockSizeInBytes = ramAccountingBlockSizeInBytes;

    this.ignoreMissing = !partitionedByColumns.isEmpty();
    DocRefResolver docRefResolver = new DocRefResolver(partitionedByColumns);

    InputFactory.Context<CollectExpression<Doc, ?>> ctx = inputFactory.ctxForRefs(txnCtx, docRefResolver);
    ctx.add(toCollect);
    expressions = ctx.expressions();
    inputRow = new InputRow(ctx.topLevelInputs());
}
 

@Nullable
public BlobShard blobShard(ShardId shardId) {
    BlobIndex blobIndex = indices.get(shardId.getIndexName());
    if (blobIndex == null) {
        return null;
    }
    return blobIndex.getShard(shardId.id());
}
 

/**
 * Return the shard location respecting global blobs data path value
 */
public File shardLocation(ShardId shardId) {
    if (blobsPath == null) {
        return new File(nodeEnvironment.availableShardPaths(shardId)[0].toFile(), BLOBS_SUB_PATH);
    }
    return shardLocation(shardId, blobsPath);
}
 

@Override
protected void shardOperationOnReplica(IndexRequest request) {
    final ShardId shardId = request.shardId();
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    indexShard.checkDiskSpace(fsService);
    final Engine.IndexingOperation operation = executeIndexRequestOnReplica(request, indexShard);
    processAfterWrite(request.refresh(), indexShard, operation.getTranslogLocation());
}
 

@Inject
public IndexVersionShardService(ShardId shardId, Settings indexSettings, IndexShard indexShard) {
  super(shardId, indexSettings);
  this.indexShard = indexShard;
  this.versioningIndexingOperationListener = new VersioningIndexingOperationListener();
  indexShard.indexingService().addListener(versioningIndexingOperationListener);
  version = new AtomicLong(System.nanoTime()); // initialise version number based on time to ensure uniqueness even if shard restarted
}
 

/**
 * Increases the primary term if {@link #increasePrimaryTerm} was called for this shard id.
 */
private IndexMetaData.Builder updatePrimaryTerm(IndexMetaData oldIndexMetaData, IndexMetaData.Builder indexMetaDataBuilder,
                                                ShardId shardId, Updates updates) {
    if (updates.increaseTerm) {
        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        indexMetaDataBuilder.primaryTerm(shardId.id(), oldIndexMetaData.primaryTerm(shardId.id()) + 1);
    }
    return indexMetaDataBuilder;
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    shardId = ShardId.readShardId(in);
    responses = new BulkItemResponse[in.readVInt()];
    for (int i = 0; i < responses.length; i++) {
        responses[i] = BulkItemResponse.readBulkItem(in);
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    shardId = ShardId.readShardId(in);
    int shardSize = in.readVInt();
    shards = new ShardStats[shardSize];
    for (int i = 0; i < shardSize; i++) {
        shards[i] = ShardStats.readShardStats(in);
    }
}
 

private void executeUpsertRequest(final UpsertByIdNode.Item item, final SettableFuture<TaskResult> futureResult) {
    ShardId shardId;
    try {
        shardId = clusterService.operationRouting().indexShards(
                clusterService.state(),
                item.index(),
                Constants.DEFAULT_MAPPING_TYPE,
                item.id(),
                item.routing()
        ).shardId();
    } catch (IndexNotFoundException e) {
        if (PartitionName.isPartition(item.index())) {
            futureResult.set(TaskResult.ZERO);
            return;
        }
        throw e;
    }

    ShardUpsertRequest upsertRequest = new ShardUpsertRequest(
            shardId, node.updateColumns(), node.insertColumns(), item.routing(), jobId());
    upsertRequest.continueOnError(false);
    ShardUpsertRequest.Item requestItem = new ShardUpsertRequest.Item(
            item.id(), item.updateAssignments(), item.insertValues(), item.version());
    upsertRequest.add(0, requestItem);

    UpsertByIdContext upsertByIdContext = new UpsertByIdContext(
            node.executionPhaseId(), upsertRequest, item, futureResult, transportShardUpsertActionDelegate);
    createJobExecutionContext(upsertByIdContext);
    try {
        jobExecutionContext.start();
    } catch (Throwable throwable) {
        futureResult.setException(throwable);
    }
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    recoveryId = in.readLong();
    shardId = ShardId.readShardId(in);
    sourceNode = DiscoveryNode.readNode(in);
    targetNode = DiscoveryNode.readNode(in);
    markAsRelocated = in.readBoolean();
    metadataSnapshot = new Store.MetadataSnapshot(in);
    recoveryType = RecoveryState.Type.fromId(in.readByte());

}