org.elasticsearch.cluster.routing.ShardIterator Java Examples

@Override
public Routing getRouting(WhereClause whereClause, @Nullable String preference) {
    Map<String, Map<String, List<Integer>>> locations = new TreeMap<>();
    GroupShardsIterator shardIterators = clusterService.operationRouting().searchShards(
            clusterService.state(),
            new String[]{index},
            null,
            preference
    );
    ShardRouting shardRouting;
    for (ShardIterator shardIterator : shardIterators) {
        shardRouting = shardIterator.nextOrNull();
        processShardRouting(locations, shardRouting, shardIterator.shardId());
    }

    return new Routing(locations);
}
 

/**
 * @param index  the name of blob-enabled index
 * @param digest sha-1 hash value of the file
 * @return null if no redirect is required, Otherwise the address to which should be redirected.
 */
public String getRedirectAddress(String index, String digest) throws MissingHTTPEndpointException {
    ShardIterator shards = clusterService.operationRouting().getShards(
        clusterService.state(), index, null, digest, "_local");

    String localNodeId = clusterService.localNode().getId();
    DiscoveryNodes nodes = clusterService.state().getNodes();
    ShardRouting shard;
    while ((shard = shards.nextOrNull()) != null) {
        if (!shard.active()) {
            continue;
        }
        if (shard.currentNodeId().equals(localNodeId)) {
            // no redirect required if the shard is on this node
            return null;
        }

        DiscoveryNode node = nodes.get(shard.currentNodeId());
        String httpAddress = node.getAttributes().get("http_address");
        if (httpAddress != null) {
            return httpAddress + "/_blobs/" + BlobIndex.stripPrefix(index) + "/" + digest;
        }
    }
    throw new MissingHTTPEndpointException("Can't find a suitable http server to serve the blob");
}
 

/**
 * Retrieves the routing for sys.shards
 * <p>
 * This routing contains ALL shards of ALL indices.
 * Any shards that are not yet assigned to a node will have a NEGATIVE shard id (see {@link UnassignedShard}
 */
public static Routing getRouting(ClusterState clusterState, RoutingProvider routingProvider, SessionContext sessionContext) {
    String[] concreteIndices = Arrays.stream(clusterState.metaData().getConcreteAllOpenIndices())
        .filter(index -> !IndexParts.isDangling(index))
        .toArray(String[]::new);
    User user = sessionContext != null ? sessionContext.user() : null;
    if (user != null) {
        List<String> accessibleTables = new ArrayList<>(concreteIndices.length);
        for (String indexName : concreteIndices) {
            String tableName = RelationName.fqnFromIndexName(indexName);
            if (user.hasAnyPrivilege(Privilege.Clazz.TABLE, tableName)) {
                accessibleTables.add(indexName);
            }
        }
        concreteIndices = accessibleTables.toArray(new String[0]);
    }

    Map<String, Map<String, IntIndexedContainer>> locations = new TreeMap<>();
    GroupShardsIterator<ShardIterator> groupShardsIterator =
        clusterState.getRoutingTable().allAssignedShardsGrouped(concreteIndices, true, true);
    for (final ShardIterator shardIt : groupShardsIterator) {
        final ShardRouting shardRouting = shardIt.nextOrNull();
        processShardRouting(clusterState.getNodes().getLocalNodeId(), locations, shardRouting, shardIt.shardId());
    }
    return new Routing(locations);
}
 

private static ShardLocation getShardLocation(String indexName,
                                              String id,
                                              @Nullable String routing,
                                              ClusterService clusterService) {
    ShardIterator shardIterator = clusterService.operationRouting().indexShards(
        clusterService.state(),
        indexName,
        id,
        routing);

    final String nodeId;
    ShardRouting shardRouting = shardIterator.nextOrNull();
    if (shardRouting == null) {
        nodeId = null;
    } else if (shardRouting.active() == false) {
        nodeId = shardRouting.relocatingNodeId();
    } else {
        nodeId = shardRouting.currentNodeId();
    }
    return new ShardLocation(shardIterator.shardId(), nodeId);
}
 

private int findReplicaLevel(ShardIterator shardIt) {
    int replicaLevel = 0;
    shardIt.reset();
    ShardRouting shard;
    while ((shard = shardIt.nextOrNull()) != null) {
        if (shard.unassigned()) {
            continue;
        }
        boolean doOnlyOnRelocating = false;
        if (shard.primary()) {
            if (shard.relocating()) {
                doOnlyOnRelocating = true;
            } else {
                continue;
            }
        }
        String nodeId = !doOnlyOnRelocating ? shard.currentNodeId() : shard.relocating() ? shard.relocatingNodeId() : null;
        if (nodeId == null) {
            continue;
        }
        replicaLevel++;
    }
    return replicaLevel;
}
 

public void start() {
    if (shardsIts.size() == 0) {
        // no shards
        try {
            listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
        } catch (Throwable e) {
            listener.onFailure(e);
        }
        return;
    }
    // count the local operations, and perform the non local ones
    int shardIndex = -1;
    for (final ShardIterator shardIt : shardsIts) {
        shardIndex++;
        final ShardRouting shard = shardIt.nextOrNull();
        if (shard != null) {
            performOperation(shardIt, shard, shardIndex);
        } else {
            // really, no shards active in this group
            onOperation(null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
        }
    }
}
 

void onFirstPhaseResult(int shardIndex, ShardRouting shard, FirstResult result, ShardIterator shardIt) {
    result.shardTarget(new SearchShardTarget(shard.currentNodeId(), shard.index(), shard.id()));
    processFirstPhaseResult(shardIndex, result);
    // we need to increment successful ops first before we compare the exit condition otherwise if we
    // are fast we could concurrently update totalOps but then preempt one of the threads which can
    // cause the successor to read a wrong value from successfulOps if second phase is very fast ie. count etc.
    successfulOps.incrementAndGet();
    // increment all the "future" shards to update the total ops since we some may work and some may not...
    // and when that happens, we break on total ops, so we must maintain them
    final int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
    if (xTotalOps == expectedTotalOps) {
        try {
            innerMoveToSecondPhase();
        } catch (Throwable e) {
            if (logger.isDebugEnabled()) {
                logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "] while moving to second phase", e);
            }
            raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
        }
    } else if (xTotalOps > expectedTotalOps) {
        raiseEarlyFailure(new IllegalStateException("unexpected higher total ops [" + xTotalOps + "] compared to expected [" + expectedTotalOps + "]"));
    }
}
 

void performFirstPhase(final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
    if (shard == null) {
        // no more active shards... (we should not really get here, but just for safety)
        onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
    } else {
        final DiscoveryNode node = nodes.get(shard.currentNodeId());
        if (node == null) {
            onFirstPhaseResult(shardIndex, shard, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
        } else {
            String[] filteringAliases = indexNameExpressionResolver.filteringAliases(clusterState, shard.index(), request.indices());
            sendExecuteFirstPhase(node, internalSearchRequest(shard, shardsIts.size(), request, filteringAliases, startTime()), new ActionListener<FirstResult>() {
                @Override
                public void onResponse(FirstResult result) {
                    onFirstPhaseResult(shardIndex, shard, result, shardIt);
                }

                @Override
                public void onFailure(Throwable t) {
                    onFirstPhaseResult(shardIndex, shard, node.id(), shardIt, t);
                }
            });
        }
    }
}
 

public void start() {
    if (expectedSuccessfulOps == 0) {
        // no search shards to search on, bail with empty response (it happens with search across _all with no indices around and consistent with broadcast operations)
        listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, buildTookInMillis(), ShardSearchFailure.EMPTY_ARRAY));
        return;
    }
    int shardIndex = -1;
    for (final ShardIterator shardIt : shardsIts) {
        shardIndex++;
        final ShardRouting shard = shardIt.nextOrNull();
        if (shard != null) {
            performFirstPhase(shardIndex, shardIt, shard);
        } else {
            // really, no shards active in this group
            onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
        }
    }
}
 

private void fillLocationsFromShardIterators(Map<String, Map<String, List<Integer>>> locations,
                                             GroupShardsIterator shardIterators,
                                             List<ShardId> missingShards) {
    ShardRouting shardRouting;
    for (ShardIterator shardIterator : shardIterators) {
        shardRouting = shardIterator.nextOrNull();
        if (shardRouting != null) {
            if (shardRouting.active()) {
                processShardRouting(locations, shardRouting);
            } else {
                missingShards.add(shardIterator.shardId());
            }
        } else {
            if (isPartitioned) {
                // if the table is partitioned maybe a index/shard just got newly created ...
                missingShards.add(shardIterator.shardId());
            } else {
                throw new UnavailableShardsException(shardIterator.shardId());
            }
        }
    }
}
 

/**
 * Retrieves the routing for sys.shards
 *
 * This routing contains ALL shards of ALL indices.
 * Any shards that are not yet assigned to a node will have a NEGATIVE shard id (see {@link UnassignedShard}
 */
@Override
public Routing getRouting(WhereClause whereClause, @Nullable String preference) {
    // TODO: filter on whereClause
    Map<String, Map<String, List<Integer>>> locations = new TreeMap<>();
    ClusterState state = service.state();
    String[] concreteIndices = state.metaData().concreteAllIndices();
    GroupShardsIterator groupShardsIterator = state.getRoutingTable().allAssignedShardsGrouped(concreteIndices, true, true);
    for (final ShardIterator shardIt : groupShardsIterator) {
        final ShardRouting shardRouting = shardIt.nextOrNull();
        processShardRouting(locations, shardRouting, shardIt.shardId());
    }
    return new Routing(locations);
}
 

@Override
protected void resolveRequest(IndexMetaData indexMetaData, StartBlobRequest request) {
    ShardIterator shardIterator = clusterService.operationRouting().indexShards(
        clusterService.state(), request.index(), request.id(), null);
    request.setShardId(shardIterator.shardId());
    super.resolveRequest(indexMetaData, request);
}
 

@Override
protected void performOperation(final ShardIterator shardIt, final ShardRouting shard, final int shardIndex) {
    if (processed.get()) {
        return;
    }
    super.performOperation(shardIt, shard, shardIndex);
}
 

@Override
protected void resolveRequest(IndexMetaData indexMetaData, PutChunkRequest request) {
    ShardIterator shardIterator = clusterService.operationRouting().indexShards(
        clusterService.state(), request.index(), null, request.digest());
    request.setShardId(shardIterator.shardId());
    super.resolveRequest(indexMetaData, request);
}
 

@Override
protected void resolveRequest(IndexMetaData indexMetaData, DeleteBlobRequest request) {
    ShardIterator shardIterator = clusterService.operationRouting()
        .indexShards(clusterService.state(), request.index(), request.id(), null);
    request.setShardId(shardIterator.shardId());
    super.resolveRequest(indexMetaData, request);
}
 

private String executionFailureMsg(@Nullable ShardRouting shard, final ShardIterator shardIt, SearchRequest request, boolean lastShard) {
    if (shard != null) {
        return shard.shortSummary() + ": Failed to execute [" + request + "] lastShard [" + lastShard + "]";
    } else {
        return shardIt.shardId() + ": Failed to execute [" + request + "] lastShard [" + lastShard + "]";
    }
}
 

private static void fillLocationsFromShardIterator(boolean ignoreMissingShards,
                                                   Map<String, Map<String, IntIndexedContainer>> locations,
                                                   ShardIterator shardIterator) {
    ShardRouting shardRouting = shardIterator.nextOrNull();
    if (shardRouting == null) {
        if (ignoreMissingShards) {
            return;
        }
        throw new UnavailableShardsException(shardIterator.shardId());
    }
    processShardRouting(locations, shardRouting);
}
 

@SuppressWarnings({"unchecked"})
void onOperation(@Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
    // we set the shard failure always, even if its the first in the replication group, and the next one
    // will work (it will just override it...)
    setFailure(shardIt, shardIndex, t);
    ShardRouting nextShard = shardIt.nextOrNull();
    if (nextShard != null) {
        if (t != null) {
            if (logger.isTraceEnabled()) {
                if (!TransportActions.isShardNotAvailableException(t)) {
                    logger.trace("{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                }
            }
        }
        performOperation(shardIt, nextShard, shardIndex);
    } else {
        if (logger.isDebugEnabled()) {
            if (t != null) {
                if (!TransportActions.isShardNotAvailableException(t)) {
                    logger.debug("{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                }
            }
        }
        if (expectedOps == counterOps.incrementAndGet()) {
            finishHim();
        }
    }
}
 

void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) {
    // we don't aggregate shard failures on non active shards (but do keep the header counts right)
    if (TransportActions.isShardNotAvailableException(t)) {
        return;
    }

    if (!(t instanceof BroadcastShardOperationFailedException)) {
        t = new BroadcastShardOperationFailedException(shardIt.shardId(), t);
    }

    Object response = shardsResponses.get(shardIndex);
    if (response == null) {
        // just override it and return
        shardsResponses.set(shardIndex, t);
    }

    if (!(response instanceof Throwable)) {
        // we should never really get here...
        return;
    }

    // the failure is already present, try and not override it with an exception that is less meaningless
    // for example, getting illegal shard state
    if (TransportActions.isReadOverrideException(t)) {
        shardsResponses.set(shardIndex, t);
    }
}
 

public Routing forIndices(ClusterState state,
                          String[] concreteIndices,
                          Map<String, Set<String>> routingValuesByIndex,
                          boolean ignoreMissingShards,
                          ShardSelection shardSelection) {

    Set<IndexShardRoutingTable> shards;
    try {
        shards = computeTargetedShards(state, concreteIndices, routingValuesByIndex);
    } catch (IndexNotFoundException e) {
        return new Routing(Collections.emptyMap());
    }
    Map<String, Map<String, IntIndexedContainer>> locations = new TreeMap<>();

    for (IndexShardRoutingTable shard : shards) {
        final ShardIterator shardIt;
        switch (shardSelection) {
            case ANY:
                if (awarenessAttributes.isEmpty()) {
                    shardIt = shard.activeInitializingShardsIt(seed);
                } else {
                    shardIt = shard.preferAttributesActiveInitializingShardsIt(
                        awarenessAttributes, state.getNodes(), seed);
                }
                break;

            case PRIMARIES:
                shardIt = shard.primaryActiveInitializingShardIt();
                break;


            default:
                throw new AssertionError("Invalid ShardSelection: " + shardSelection);
        }
        fillLocationsFromShardIterator(ignoreMissingShards, locations, shardIt);
    }
    return new Routing(locations);
}
 

@Nullable
private ShardLocation getShardLocation(String indexName, String id, @Nullable String routing) {
    try {
        ShardIterator shardIterator = clusterService.operationRouting().indexShards(
            clusterService.state(),
            indexName,
            id,
            routing
        );

        final String nodeId;
        ShardRouting shardRouting = shardIterator.nextOrNull();
        if (shardRouting == null) {
            nodeId = null;
        } else if (shardRouting.active() == false) {
            nodeId = shardRouting.relocatingNodeId();
        } else {
            nodeId = shardRouting.currentNodeId();
        }
        if (nodeId == null && LOGGER.isDebugEnabled()) {
            LOGGER.debug("Unable to get the node id for index {} and shard {}", indexName, id);
        }
        return new ShardLocation(shardIterator.shardId(), nodeId);
    } catch (IndexNotFoundException e) {
        if (!autoCreateIndices) {
            throw e;
        }
        return null;
    }
}
 

protected ShardIterator shards(ClusterState clusterState, IngestReplicaShardRequest request) {
    return clusterState.routingTable().index(request.index()).shard(request.shardId().id()).shardsIt();
}
 

private ShardId localShardId(String index, String digest) {
    ShardIterator si = clusterService.operationRouting().getShards(
        clusterService.state(), index, null, digest, "_only_local");
    return si.shardId();
}
 

@Override
protected void resolveRequest(MetaData metaData, String concreteIndex, DeleteBlobRequest request) {
    ShardIterator shardIterator = clusterService.operationRouting()
            .indexShards(clusterService.state(), concreteIndex, null, request.id(), null);
    request.setShardId(shardIterator.shardId());
}
 

@Override
protected void resolveRequest(MetaData metaData, String concreteIndex, PutChunkRequest request) {
    ShardIterator shardIterator = clusterService.operationRouting().indexShards(
            clusterService.state(), concreteIndex, null, null, request.digest());
    request.setShardId(shardIterator.shardId());
}
 

public int findQuorum(ClusterState clusterState, ShardIterator shardIt, IngestLeaderShardRequest request) {
    if (request.requiredConsistency() == Consistency.IGNORE) {
        return 0;
    }
    int numberOfDataNodes = 0;
    for (DiscoveryNode node : clusterState.getNodes()) {
        if (node.isDataNode()) {
            numberOfDataNodes++;
        }
    }
    // single node, do not care about replica
    if (numberOfDataNodes == 1) {
        return 0;
    }
    int replicaLevelOfIndex = clusterState.metaData().index(request.index()).getNumberOfReplicas();
    // no replica defined, so nothing to check
    if (replicaLevelOfIndex == 0) {
        return 0;
    }
    int replicaLevel = findReplicaLevel(shardIt) + 1;
    switch (request.requiredConsistency()) {
        case ONE:
            if (replicaLevel >= 1 && replicaLevelOfIndex >= 1) {
                return 1;
            }
            break;
        case QUORUM:
            int quorum = (replicaLevelOfIndex / 2) + 1;
            if (replicaLevel >= quorum) {
                return quorum;
            }
            break;
        case ALL:
            if (replicaLevel == replicaLevelOfIndex) {
                return replicaLevel;
            }
            break;
    }
    // quorum not matched - we have a problem
    return -1;
}
 

protected ShardIterator shards(ClusterState clusterState, IngestLeaderShardRequest request) {
    return clusterState.routingTable().index(request.index()).shard(request.getShardId().id()).shardsIt();
}
 

@Override
protected ShardIterator shards(ClusterState state, InternalRequest request) {
    return clusterService.operationRouting()
            .getShards(state, request.request().index(), request.request().shardId(), request.request().preference());
}
 

@Override
protected ShardIterator shards(ClusterState state, InternalRequest request) {
    return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().type(), request.request().id(),
            request.request().routing(), request.request().preference());
}
 

@Override
protected ShardIterator shards(ClusterState state, InternalRequest request) {
    return clusterService.operationRouting()
            .getShards(state, request.concreteIndex(), request.request().shardId(), request.request().preference());
}