Java Code Examples for org.elasticsearch.cluster.routing.ShardRouting#relocating()

private void recover(StartRecoveryRequest request, ActionListener<RecoveryResponse> listener) throws IOException {
    final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
    final IndexShard shard = indexService.getShard(request.shardId().id());

    final ShardRouting routingEntry = shard.routingEntry();

    if (routingEntry.primary() == false || routingEntry.active() == false) {
        throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary");
    }

    if (request.isPrimaryRelocation()
        && (
            routingEntry.relocating() == false
            || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) {
        LOGGER.debug(
            "delaying recovery of {} as source shard is not marked yet as relocating to {}",
            request.shardId(), request.targetNode());
        throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]");
    }

    RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard);
    LOGGER.trace(
        "[{}][{}] starting recovery to {}",
        request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode());
    handler.recoverToTarget(ActionListener.runAfter(listener, () -> ongoingRecoveries.remove(shard, handler)));
}
 

private int findReplicaLevel(ShardIterator shardIt) {
    int replicaLevel = 0;
    shardIt.reset();
    ShardRouting shard;
    while ((shard = shardIt.nextOrNull()) != null) {
        if (shard.unassigned()) {
            continue;
        }
        boolean doOnlyOnRelocating = false;
        if (shard.primary()) {
            if (shard.relocating()) {
                doOnlyOnRelocating = true;
            } else {
                continue;
            }
        }
        String nodeId = !doOnlyOnRelocating ? shard.currentNodeId() : shard.relocating() ? shard.relocatingNodeId() : null;
        if (nodeId == null) {
            continue;
        }
        replicaLevel++;
    }
    return replicaLevel;
}
 

private static ClusterAllocationExplanation explainShard(ShardRouting shardRouting,
                                                         RoutingAllocation allocation,
                                                         GatewayAllocator gatewayAllocator,
                                                         ShardsAllocator shardAllocator) {
    allocation.setDebugMode(RoutingAllocation.DebugMode.EXCLUDE_YES_DECISIONS);

    ShardAllocationDecision shardDecision;
    if (shardRouting.initializing() || shardRouting.relocating()) {
        shardDecision = ShardAllocationDecision.NOT_TAKEN;
    } else {
        AllocateUnassignedDecision allocateDecision = shardRouting.unassigned() ?
            gatewayAllocator.decideUnassignedShardAllocation(shardRouting, allocation) : AllocateUnassignedDecision.NOT_TAKEN;
        if (allocateDecision.isDecisionTaken() == false) {
            shardDecision = shardAllocator.decideShardAllocation(shardRouting, allocation);
        } else {
            shardDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN);
        }
    }
    return new ClusterAllocationExplanation(
        shardRouting,
        shardRouting.currentNodeId() != null ? allocation.nodes().get(shardRouting.currentNodeId()) : null,
        shardRouting.relocatingNodeId() != null ? allocation.nodes().get(shardRouting.relocatingNodeId()) : null,
        null,
        shardDecision
    );
}
 

private void addTrackingInfo(IndexShardRoutingTable indexShardRoutingTable, ShardRouting primaryShard, Set<String> trackedShards,
                             Set<String> untrackedShards) {
    for (ShardRouting shr : indexShardRoutingTable.shards()) {
        if (shr.unassigned() == false) {
            if (shr.initializing()) {
                if (randomBoolean()) {
                    trackedShards.add(shr.allocationId().getId());
                } else {
                    untrackedShards.add(shr.allocationId().getId());
                }
            } else {
                trackedShards.add(shr.allocationId().getId());
                if (shr.relocating()) {
                    if (primaryShard == shr.getTargetRelocatingShard() || randomBoolean()) {
                        trackedShards.add(shr.getTargetRelocatingShard().allocationId().getId());
                    } else {
                        untrackedShards.add(shr.getTargetRelocatingShard().allocationId().getId());
                    }
                }
            }
        }
    }
}
 

private Set<ShardRouting> getExpectedReplicas(ShardId shardId, ClusterState state, Set<String> trackedShards) {
    Set<ShardRouting> expectedReplicas = new HashSet<>();
    String localNodeId = state.nodes().getLocalNodeId();
    if (state.routingTable().hasIndex(shardId.getIndexName())) {
        for (ShardRouting shardRouting : state.routingTable().shardRoutingTable(shardId)) {
            if (shardRouting.unassigned()) {
                continue;
            }
            if (localNodeId.equals(shardRouting.currentNodeId()) == false) {
                if (trackedShards.contains(shardRouting.allocationId().getId())) {
                    expectedReplicas.add(shardRouting);
                }
            }

            if (shardRouting.relocating() && localNodeId.equals(shardRouting.relocatingNodeId()) == false) {
                if (trackedShards.contains(shardRouting.getTargetRelocatingShard().allocationId().getId())) {
                    expectedReplicas.add(shardRouting.getTargetRelocatingShard());
                }
            }
        }
    }
    return expectedReplicas;
}
 

@Override
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
    IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index());
    int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, DEFAULT_SHARD_LIMIT);
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0",
                indexShardLimit, clusterShardLimit);
    }

    int indexShardCount = 0;
    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
        if (nodeShard.index().equals(shardRouting.index())) {
            indexShardCount++;
        }
    }
    if (clusterShardLimit > 0 && nodeShardCount >= clusterShardLimit) {
        return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]",
                nodeShardCount, clusterShardLimit);
    }
    if (indexShardLimit > 0 && indexShardCount >= indexShardLimit) {
        return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]",
                shardRouting.index(), indexShardCount, indexShardLimit);
    }
    return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node",
            indexShardLimit, clusterShardLimit);
}
 

@Override
public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
    IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index());
    int indexShardLimit = indexMd.getSettings().getAsInt(INDEX_TOTAL_SHARDS_PER_NODE, -1);
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [index: %d, cluster: %d] <= 0",
                indexShardLimit, clusterShardLimit);
    }

    int indexShardCount = 0;
    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
        if (nodeShard.index().equals(shardRouting.index())) {
            indexShardCount++;
        }
    }
    // Subtle difference between the `canAllocate` and `canRemain` is that
    // this checks > while canAllocate checks >=
    if (clusterShardLimit > 0 && nodeShardCount > clusterShardLimit) {
        return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]",
                nodeShardCount, clusterShardLimit);
    }
    if (indexShardLimit > 0 && indexShardCount > indexShardLimit) {
        return allocation.decision(Decision.NO, NAME, "too many shards for this index [%s] on node [%d], limit: [%d]",
                shardRouting.index(), indexShardCount, indexShardLimit);
    }
    return allocation.decision(Decision.YES, NAME, "shard count under index limit [%d] and node limit [%d] of total shards per node",
            indexShardLimit, clusterShardLimit);
}
 

@Override
public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
    // Only checks the node-level limit, not the index-level
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limit disabled: [cluster: %d] <= 0",
                clusterShardLimit);
    }

    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
    }
    if (clusterShardLimit >= 0 && nodeShardCount >= clusterShardLimit) {
        return allocation.decision(Decision.NO, NAME, "too many shards for this node [%d], limit: [%d]",
                nodeShardCount, clusterShardLimit);
    }
    return allocation.decision(Decision.YES, NAME, "shard count under node limit [%d] of total shards per node",
            clusterShardLimit);
}
 

/**
 * Returns the size of all shards that are currently being relocated to
 * the node, but may not be finished transfering yet.
 *
 * If subtractShardsMovingAway is set then the size of shards moving away is subtracted from the total size
 * of all shards
 */
public static long sizeOfRelocatingShards(RoutingNode node, ClusterInfo clusterInfo, boolean subtractShardsMovingAway, String dataPath) {
    long totalSize = 0;
    for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING)) {
        String actualPath = clusterInfo.getDataPath(routing);
        if (dataPath.equals(actualPath)) {
            if (routing.initializing() && routing.relocatingNodeId() != null) {
                totalSize += getShardSize(routing, clusterInfo);
            } else if (subtractShardsMovingAway && routing.relocating()) {
                totalSize -= getShardSize(routing, clusterInfo);
            }
        }
    }
    return totalSize;
}
 

public ClusterShardHealth(int shardId, final IndexShardRoutingTable shardRoutingTable) {
    this.shardId = shardId;
    for (ShardRouting shardRouting : shardRoutingTable) {
        if (shardRouting.active()) {
            activeShards++;
            if (shardRouting.relocating()) {
                // the shard is relocating, the one it is relocating to will be in initializing state, so we don't count it
                relocatingShards++;
            }
            if (shardRouting.primary()) {
                primaryActive = true;
            }
        } else if (shardRouting.initializing()) {
            initializingShards++;
        } else if (shardRouting.unassigned()) {
            unassignedShards++;
        }
    }
    if (primaryActive) {
        if (activeShards == shardRoutingTable.size()) {
            status = ClusterHealthStatus.GREEN;
        } else {
            status = ClusterHealthStatus.YELLOW;
        }
    } else {
        status = ClusterHealthStatus.RED;
    }
}
 

private Decision doDecide(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
                          BiPredicate<Integer, Integer> decider) {
    IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
    final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limits are disabled: [index: %d, cluster: %d] <= 0",
                indexShardLimit, clusterShardLimit);
    }

    int indexShardCount = 0;
    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
        if (nodeShard.index().equals(shardRouting.index())) {
            indexShardCount++;
        }
    }

    if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
            nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
    }
    if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
            indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
    }
    return allocation.decision(Decision.YES, NAME,
        "the shard count [%d] for this node is under the index limit [%d] and cluster level node limit [%d]",
        nodeShardCount, indexShardLimit, clusterShardLimit);
}
 

@Override
public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
    // Only checks the node-level limit, not the index-level
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limits are disabled: [cluster: %d] <= 0",
                clusterShardLimit);
    }

    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
    }
    if (nodeShardCount >= clusterShardLimit) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
            nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
    }
    return allocation.decision(Decision.YES, NAME,
        "the shard count [%d] for this node is under the cluster level node limit [%d]",
        nodeShardCount, clusterShardLimit);
}
 

public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable) {
    this.shardId = shardId;
    int computeActiveShards = 0;
    int computeRelocatingShards = 0;
    int computeInitializingShards = 0;
    int computeUnassignedShards = 0;
    for (ShardRouting shardRouting : shardRoutingTable) {
        if (shardRouting.active()) {
            computeActiveShards++;
            if (shardRouting.relocating()) {
                // the shard is relocating, the one it is relocating to will be in initializing state, so we don't count it
                computeRelocatingShards++;
            }
        } else if (shardRouting.initializing()) {
            computeInitializingShards++;
        } else if (shardRouting.unassigned()) {
            computeUnassignedShards++;
        }
    }
    ClusterHealthStatus computeStatus;
    final ShardRouting primaryRouting = shardRoutingTable.primaryShard();
    if (primaryRouting.active()) {
        if (computeActiveShards == shardRoutingTable.size()) {
            computeStatus = ClusterHealthStatus.GREEN;
        } else {
            computeStatus = ClusterHealthStatus.YELLOW;
        }
    } else {
        computeStatus = getInactivePrimaryHealth(primaryRouting);
    }
    this.status = computeStatus;
    this.activeShards = computeActiveShards;
    this.relocatingShards = computeRelocatingShards;
    this.initializingShards = computeInitializingShards;
    this.unassignedShards = computeUnassignedShards;
    this.primaryActive = primaryRouting.active();
}
 

private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
    if (awarenessAttributes.length == 0) {
        return allocation.decision(Decision.YES, NAME, "no allocation awareness enabled");
    }

    IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.index());
    int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary
    for (String awarenessAttribute : awarenessAttributes) {
        // the node the shard exists on must be associated with an awareness attribute
        if (!node.node().attributes().containsKey(awarenessAttribute)) {
            return allocation.decision(Decision.NO, NAME, "node does not contain awareness attribute: [%s]", awarenessAttribute);
        }

        // build attr_value -> nodes map
        ObjectIntHashMap<String> nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute);

        // build the count of shards per attribute value
        ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
        for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting)) {
            if (assignedShard.started() || assignedShard.initializing()) {
                // Note: this also counts relocation targets as that will be the new location of the shard.
                // Relocation sources should not be counted as the shard is moving away
                RoutingNode routingNode = allocation.routingNodes().node(assignedShard.currentNodeId());
                shardPerAttribute.addTo(routingNode.node().attributes().get(awarenessAttribute), 1);
            }
        }

        if (moveToNode) {
            if (shardRouting.assignedToNode()) {
                String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId();
                if (!node.nodeId().equals(nodeId)) {
                    // we work on different nodes, move counts around
                    shardPerAttribute.putOrAdd(allocation.routingNodes().node(nodeId).node().attributes().get(awarenessAttribute), 0, -1);
                    shardPerAttribute.addTo(node.node().attributes().get(awarenessAttribute), 1);
                }
            } else {
                shardPerAttribute.addTo(node.node().attributes().get(awarenessAttribute), 1);
            }
        }

        int numberOfAttributes = nodesPerAttribute.size();
        String[] fullValues = forcedAwarenessAttributes.get(awarenessAttribute);
        if (fullValues != null) {
            for (String fullValue : fullValues) {
                if (!shardPerAttribute.containsKey(fullValue)) {
                    numberOfAttributes++;
                }
            }
        }
        // TODO should we remove ones that are not part of full list?

        int averagePerAttribute = shardCount / numberOfAttributes;
        int totalLeftover = shardCount % numberOfAttributes;
        int requiredCountPerAttribute;
        if (averagePerAttribute == 0) {
            // if we have more attributes values than shard count, no leftover
            totalLeftover = 0;
            requiredCountPerAttribute = 1;
        } else {
            requiredCountPerAttribute = averagePerAttribute;
        }
        int leftoverPerAttribute = totalLeftover == 0 ? 0 : 1;

        int currentNodeCount = shardPerAttribute.get(node.node().attributes().get(awarenessAttribute));
        // if we are above with leftover, then we know we are not good, even with mod
        if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                    "too many shards on node for attribute: [%s], required per attribute: [%d], node count: [%d], leftover: [%d]",
                    awarenessAttribute, requiredCountPerAttribute, currentNodeCount, leftoverPerAttribute);
        }
        // all is well, we are below or same as average
        if (currentNodeCount <= requiredCountPerAttribute) {
            continue;
        }
    }

    return allocation.decision(Decision.YES, NAME, "node meets awareness requirements");
}
 

@Override
public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) {
    DiscoveryNode discoNode = allocation.nodes().resolveNode(node);
    boolean found = false;
    for (RoutingNodes.RoutingNodeIterator it = allocation.routingNodes().routingNodeIter(discoNode.id()); it.hasNext(); ) {
        ShardRouting shardRouting = it.next();
        if (!shardRouting.shardId().equals(shardId)) {
            continue;
        }
        found = true;
        if (shardRouting.relocatingNodeId() != null) {
            if (shardRouting.initializing()) {
                // the shard is initializing and recovering from another node, simply cancel the recovery
                it.remove();
                // and cancel the relocating state from the shard its being relocated from
                RoutingNode relocatingFromNode = allocation.routingNodes().node(shardRouting.relocatingNodeId());
                if (relocatingFromNode != null) {
                    for (ShardRouting fromShardRouting : relocatingFromNode) {
                        if (fromShardRouting.isSameShard(shardRouting) && fromShardRouting.state() == RELOCATING) {
                            allocation.routingNodes().cancelRelocation(fromShardRouting);
                            break;
                        }
                    }
                }
            } else if (shardRouting.relocating()) {

                // the shard is relocating to another node, cancel the recovery on the other node, and deallocate this one
                if (!allowPrimary && shardRouting.primary()) {
                    // can't cancel a primary shard being initialized
                    if (explain) {
                        return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
                                "can't cancel " + shardId + " on node " + discoNode + ", shard is primary and initializing its state"));
                    }
                    throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
                            discoNode + ", shard is primary and initializing its state");
                }
                it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
                // now, go and find the shard that is initializing on the target node, and cancel it as well...
                RoutingNodes.RoutingNodeIterator initializingNode = allocation.routingNodes().routingNodeIter(shardRouting.relocatingNodeId());
                if (initializingNode != null) {
                    while (initializingNode.hasNext()) {
                        ShardRouting initializingShardRouting = initializingNode.next();
                        if (initializingShardRouting.isRelocationTargetOf(shardRouting)) {
                            initializingNode.remove();
                        }
                    }
                }
            }
        } else {
            // the shard is not relocating, its either started, or initializing, just cancel it and move on...
            if (!allowPrimary && shardRouting.primary()) {
                // can't cancel a primary shard being initialized
                if (explain) {
                    return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
                            "can't cancel " + shardId + " on node " + discoNode + ", shard is primary and started"));
                }
                throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
                        discoNode + ", shard is primary and started");
            }
            it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
        }
    }
    if (!found) {
        if (explain) {
            return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
                    "can't cancel " + shardId + ", failed to find it on node " + discoNode));
        }
        throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + ", failed to find it on node " + discoNode);
    }
    return new RerouteExplanation(this, allocation.decision(Decision.YES, "cancel_allocation_command",
            "shard " + shardId + " on node " + discoNode + " can be cancelled"));
}
 

private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
    if (awarenessAttributes.isEmpty()) {
        return allocation.decision(Decision.YES, NAME,
            "allocation awareness is not enabled, set cluster setting [%s] to enable it",
            CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey());
    }

    IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
    int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary
    for (String awarenessAttribute : awarenessAttributes) {
        // the node the shard exists on must be associated with an awareness attribute
        if (!node.node().getAttributes().containsKey(awarenessAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]",
                awarenessAttribute, CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
                allocation.debugDecision() ? Strings.collectionToCommaDelimitedString(awarenessAttributes) : null);
        }

        // build attr_value -> nodes map
        ObjectIntHashMap<String> nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute);

        // build the count of shards per attribute value
        ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
        for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting.shardId())) {
            if (assignedShard.started() || assignedShard.initializing()) {
                // Note: this also counts relocation targets as that will be the new location of the shard.
                // Relocation sources should not be counted as the shard is moving away
                RoutingNode routingNode = allocation.routingNodes().node(assignedShard.currentNodeId());
                shardPerAttribute.addTo(routingNode.node().getAttributes().get(awarenessAttribute), 1);
            }
        }

        if (moveToNode) {
            if (shardRouting.assignedToNode()) {
                String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId();
                if (!node.nodeId().equals(nodeId)) {
                    // we work on different nodes, move counts around
                    shardPerAttribute.putOrAdd(allocation.routingNodes().node(nodeId).node().getAttributes().get(awarenessAttribute),
                            0, -1);
                    shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1);
                }
            } else {
                shardPerAttribute.addTo(node.node().getAttributes().get(awarenessAttribute), 1);
            }
        }

        int numberOfAttributes = nodesPerAttribute.size();
        List<String> fullValues = forcedAwarenessAttributes.get(awarenessAttribute);
        if (fullValues != null) {
            for (String fullValue : fullValues) {
                if (!shardPerAttribute.containsKey(fullValue)) {
                    numberOfAttributes++;
                }
            }
        }
        // TODO should we remove ones that are not part of full list?

        int averagePerAttribute = shardCount / numberOfAttributes;
        int totalLeftover = shardCount % numberOfAttributes;
        int requiredCountPerAttribute;
        if (averagePerAttribute == 0) {
            // if we have more attributes values than shard count, no leftover
            totalLeftover = 0;
            requiredCountPerAttribute = 1;
        } else {
            requiredCountPerAttribute = averagePerAttribute;
        }
        int leftoverPerAttribute = totalLeftover == 0 ? 0 : 1;

        int currentNodeCount = shardPerAttribute.get(node.node().getAttributes().get(awarenessAttribute));
        // if we are above with leftover, then we know we are not good, even with mod
        if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
            return allocation.decision(Decision.NO, NAME,
                    "there are too many copies of the shard allocated to nodes with attribute [%s], there are [%d] total configured " +
                    "shard copies for this shard id and [%d] total attribute values, expected the allocated shard count per " +
                    "attribute [%d] to be less than or equal to the upper bound of the required number of shards per attribute [%d]",
                    awarenessAttribute,
                    shardCount,
                    numberOfAttributes,
                    currentNodeCount,
                    requiredCountPerAttribute + leftoverPerAttribute);
        }
        // all is well, we are below or same as average
        if (currentNodeCount <= requiredCountPerAttribute) {
            continue;
        }
    }

    return allocation.decision(Decision.YES, NAME, "node meets all awareness attribute requirements");
}