Java Code Examples for org.apache.helix.model.StateModelDefinition#getStateCountMap()

private void validateMap(String resource, String partition, int replica,
    Map<String, String> instanceStateMap, StateModelDefinition stateModelDef) {
  if (instanceStateMap == null || instanceStateMap.isEmpty()) {
    _testSuccess = false;
    Assert.fail(
        String.format("Resource %s partition %s has no active replica!", resource, partition));
  }
  if (instanceStateMap.size() < replica) {
    _testSuccess = false;
    Assert.fail(
        String.format("Resource %s partition %s has %d active replica, less than required %d!",
            resource, partition, instanceStateMap.size(), replica));
  }

  Map<String, Integer> stateCountMap = stateModelDef.getStateCountMap(NUM_NODE, replica);
  String topState = stateModelDef.getStatesPriorityList().get(0);
  if (stateCountMap.get(topState) == 1) {
    int topCount = 0;
    for (String val : instanceStateMap.values()) {
      if (topState.equals(val)) {
        topCount++;
      }
    }
    if (topCount > 1) {
      _testSuccess = false;
      Assert.fail(String.format("Resource %s partition %s has %d replica in %s, more than 1!",
          resource, partition, topCount, topState));
    }
  }
}
 

/**
 * Validate there should be always minimal active replica and top state replica for each
 * partition.
 * Also make sure there is always some partitions with only active replica count.
 */
protected void validateMinActiveAndTopStateReplica(IdealState is, ExternalView ev,
    int minActiveReplica, int numNodes) {
  StateModelDefinition stateModelDef =
      BuiltInStateModelDefinitions.valueOf(is.getStateModelDefRef()).getStateModelDefinition();
  String topState = stateModelDef.getStatesPriorityList().get(0);
  int replica = Integer.valueOf(is.getReplicas());

  Map<String, Integer> stateCount = stateModelDef.getStateCountMap(numNodes, replica);
  Set<String> activeStates = stateCount.keySet();

  for (String partition : is.getPartitionSet()) {
    Map<String, String> assignmentMap = ev.getRecord().getMapField(partition);
    Assert.assertNotNull(assignmentMap,
        is.getResourceName() + "'s best possible assignment is null for partition " + partition);
    Assert.assertTrue(!assignmentMap.isEmpty(),
        is.getResourceName() + "'s partition " + partition + " has no best possible map in IS.");

    boolean hasTopState = false;
    int activeReplica = 0;
    for (String state : assignmentMap.values()) {
      if (topState.equalsIgnoreCase(state)) {
        hasTopState = true;
      }
      if (activeStates.contains(state)) {
        activeReplica++;
      }
    }

    if (activeReplica < minActiveReplica) {
      int a = 0;
    }

    Assert.assertTrue(hasTopState, String.format("%s missing %s replica", partition, topState));
    Assert.assertTrue(activeReplica >= minActiveReplica,
        String.format("%s has less active replica %d then required %d", partition, activeReplica,
            minActiveReplica));
  }
}
 

/**
 * For a partition, given its preferenceList, bestPossibleState, and currentState, determine which
 * type of rebalance is needed to model IdealState's states defined by the state model definition.
 * @return RebalanceType needed to bring the replicas to idea states
 *         RECOVERY_BALANCE - not all required states (replicas) are available through all
 *         replicas, or the partition is disabled
 *         NONE - current state matches the ideal state
 *         LOAD_BALANCE - although all replicas required exist, Helix needs to optimize the
 *         allocation
 */
private RebalanceType getRebalanceType(ResourceControllerDataProvider cache,
    Map<String, String> bestPossibleMap, List<String> preferenceList,
    StateModelDefinition stateModelDef, Map<String, String> currentStateMap,
    IdealState idealState, String partitionName) {
  if (preferenceList == null) {
    preferenceList = Collections.emptyList();
  }

  // If there is a minimum active replica number specified in IS, we should respect it.
  // TODO: We should implement the per replica level throttling with generated message
  // Issue: https://github.com/apache/helix/issues/343
  int replica = idealState.getMinActiveReplicas() == -1
      ? idealState.getReplicaCount(preferenceList.size())
      : idealState.getMinActiveReplicas();
  Set<String> activeList = new HashSet<>(preferenceList);
  activeList.retainAll(cache.getEnabledLiveInstances());

  // For each state, check that this partition currently has the required number of that state as
  // required by StateModelDefinition.
  LinkedHashMap<String, Integer> expectedStateCountMap =
      stateModelDef.getStateCountMap(activeList.size(), replica); // StateModelDefinition's counts
  // Current counts without disabled partitions or disabled instances
  Map<String, String> currentStateMapWithoutDisabled = new HashMap<>(currentStateMap);
  currentStateMapWithoutDisabled.keySet().removeAll(
      cache.getDisabledInstancesForPartition(idealState.getResourceName(), partitionName));
  Map<String, Integer> currentStateCounts =
      StateModelDefinition.getStateCounts(currentStateMapWithoutDisabled);

  // Go through each state and compare counts
  for (String state : expectedStateCountMap.keySet()) {
    Integer expectedCount = expectedStateCountMap.get(state);
    Integer currentCount = currentStateCounts.get(state);
    expectedCount = expectedCount == null ? 0 : expectedCount;
    currentCount = currentCount == null ? 0 : currentCount;

    // If counts do not match up, this partition requires recovery
    if (currentCount < expectedCount) {
      // Recovery is not needed in cases where this partition just started, was dropped, or is in
      // error
      if (!state.equals(HelixDefinedState.DROPPED.name())
          && !state.equals(HelixDefinedState.ERROR.name())
          && !state.equals(stateModelDef.getInitialState())) {
        return RebalanceType.RECOVERY_BALANCE;
      }
    }
  }
  // No recovery needed, all expected replicas exist
  // Check if this partition is actually in the BestPossibleState
  if (currentStateMap.equals(bestPossibleMap)) {
    return RebalanceType.NONE; // No further action required
  } else {
    return RebalanceType.LOAD_BALANCE; // Required state counts are satisfied, but in order to
    // achieve BestPossibleState, load balance may be required
    // to shift replicas around
  }
}
 

@Override
public IdealState computeNewIdealState(String resourceName,
    IdealState currentIdealState, CurrentStateOutput currentStateOutput,
    ResourceControllerDataProvider clusterData) {

  IdealState cachedIdealState = getCachedIdealState(resourceName, clusterData);
  if (cachedIdealState != null) {
    LOG.debug("Use cached IdealState for " + resourceName);
    return cachedIdealState;
  }

  LOG.info("Computing IdealState for " + resourceName);

  List<String> partitions = getStablePartitionList(clusterData, currentIdealState);
  String stateModelName = currentIdealState.getStateModelDefRef();
  StateModelDefinition stateModelDef = clusterData.getStateModelDef(stateModelName);
  if (stateModelDef == null) {
    LOG.error("State Model Definition null for resource: " + resourceName);
    throw new HelixException("State Model Definition null for resource: " + resourceName);
  }
  Map<String, LiveInstance> liveInstance = clusterData.getLiveInstances();
  int replicas = currentIdealState.getReplicaCount(liveInstance.size());

  LinkedHashMap<String, Integer> stateCountMap = stateModelDef
      .getStateCountMap(liveInstance.size(), replicas);
  List<String> liveNodes = new ArrayList<>(liveInstance.keySet());
  List<String> allNodes = new ArrayList<>(clusterData.getAllInstances());
  allNodes.removeAll(clusterData.getDisabledInstances());
  liveNodes.retainAll(allNodes);

  Map<String, Map<String, String>> currentMapping =
      currentMapping(currentStateOutput, resourceName, partitions, stateCountMap);

  // If there are nodes tagged with resource name, use only those nodes
  Set<String> taggedNodes = new HashSet<String>();
  Set<String> taggedLiveNodes = new HashSet<String>();
  if (currentIdealState.getInstanceGroupTag() != null) {
    for (String instanceName : allNodes) {
      if (clusterData.getInstanceConfigMap().get(instanceName)
          .containsTag(currentIdealState.getInstanceGroupTag())) {
        taggedNodes.add(instanceName);
        if (liveInstance.containsKey(instanceName)) {
          taggedLiveNodes.add(instanceName);
        }
      }
    }
    if (!taggedLiveNodes.isEmpty()) {
      // live nodes exist that have this tag
      if (LOG.isInfoEnabled()) {
        LOG.info("found the following participants with tag "
            + currentIdealState.getInstanceGroupTag() + " for " + resourceName + ": "
            + taggedLiveNodes);
      }
    } else if (taggedNodes.isEmpty()) {
      // no live nodes and no configured nodes have this tag
      LOG.warn("Resource " + resourceName + " has tag " + currentIdealState.getInstanceGroupTag()
          + " but no configured participants have this tag");
    } else {
      // configured nodes have this tag, but no live nodes have this tag
      LOG.warn("Resource " + resourceName + " has tag " + currentIdealState.getInstanceGroupTag()
          + " but no live participants have this tag");
    }
    allNodes = new ArrayList<>(taggedNodes);
    liveNodes = new ArrayList<>(taggedLiveNodes);
  }

  // sort node lists to ensure consistent preferred assignments
  Collections.sort(allNodes);
  Collections.sort(liveNodes);

  int maxPartition = currentIdealState.getMaxPartitionsPerInstance();
  _rebalanceStrategy =
      getRebalanceStrategy(currentIdealState.getRebalanceStrategy(), partitions, resourceName,
          stateCountMap, maxPartition);
  ZNRecord newMapping = _rebalanceStrategy
      .computePartitionAssignment(allNodes, liveNodes, currentMapping, clusterData);

  if (LOG.isDebugEnabled()) {
    LOG.debug("currentMapping: " + currentMapping);
    LOG.debug("stateCountMap: " + stateCountMap);
    LOG.debug("liveNodes: " + liveNodes);
    LOG.debug("allNodes: " + allNodes);
    LOG.debug("maxPartition: " + maxPartition);
    LOG.debug("newMapping: " + newMapping);
  }

  IdealState newIdealState = new IdealState(resourceName);
  newIdealState.getRecord().setSimpleFields(currentIdealState.getRecord().getSimpleFields());
  newIdealState.setRebalanceMode(RebalanceMode.FULL_AUTO);
  newIdealState.getRecord().setListFields(newMapping.getListFields());

  return newIdealState;
}
 

private Map<String, String> computePartitionHealth(String clusterId, String resourceName) {
  HelixAdmin admin = getHelixAdmin();
  IdealState idealState = admin.getResourceIdealState(clusterId, resourceName);
  ExternalView externalView = admin.getResourceExternalView(clusterId, resourceName);
  StateModelDefinition stateModelDef =
      admin.getStateModelDef(clusterId, idealState.getStateModelDefRef());
  String initialState = stateModelDef.getInitialState();
  List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
  statesPriorityList = statesPriorityList.subList(0, statesPriorityList.indexOf(initialState)); // Trim
                                                                                                // stateList
                                                                                                // to
                                                                                                // initialState
                                                                                                // and
                                                                                                // above
  int minActiveReplicas = idealState.getMinActiveReplicas();

  // Start the logic that determines the health status of each partition
  Map<String, String> partitionHealthResult = new HashMap<>();
  Set<String> allPartitionNames = idealState.getPartitionSet();
  if (!allPartitionNames.isEmpty()) {
    for (String partitionName : allPartitionNames) {
      int replicaCount =
          idealState.getReplicaCount(idealState.getPreferenceList(partitionName).size());
      // Simplify expectedStateCountMap by assuming that all instances are available to reduce
      // computation load on this REST endpoint
      LinkedHashMap<String, Integer> expectedStateCountMap =
          stateModelDef.getStateCountMap(replicaCount, replicaCount);
      // Extract all states into Collections from ExternalView
      Map<String, String> stateMapInExternalView = externalView.getStateMap(partitionName);
      Collection<String> allReplicaStatesInExternalView =
          (stateMapInExternalView != null && !stateMapInExternalView.isEmpty())
              ? stateMapInExternalView.values()
              : Collections.<String> emptyList();
      int numActiveReplicasInExternalView = 0;
      HealthStatus status = HealthStatus.HEALTHY;

      // Go through all states that are "active" states (higher priority than InitialState)
      for (int statePriorityIndex = 0; statePriorityIndex < statesPriorityList
          .size(); statePriorityIndex++) {
        String currentState = statesPriorityList.get(statePriorityIndex);
        int currentStateCountInIdealState = expectedStateCountMap.get(currentState);
        int currentStateCountInExternalView =
            Collections.frequency(allReplicaStatesInExternalView, currentState);
        numActiveReplicasInExternalView += currentStateCountInExternalView;
        // Top state counts must match, if not, unhealthy
        if (statePriorityIndex == 0
            && currentStateCountInExternalView != currentStateCountInIdealState) {
          status = HealthStatus.UNHEALTHY;
          break;
        } else if (currentStateCountInExternalView < currentStateCountInIdealState) {
          // For non-top states, if count in ExternalView is less than count in IdealState,
          // partially healthy
          status = HealthStatus.PARTIAL_HEALTHY;
        }
      }
      if (numActiveReplicasInExternalView < minActiveReplicas) {
        // If this partition does not satisfy the number of minimum active replicas, unhealthy
        status = HealthStatus.UNHEALTHY;
      }
      partitionHealthResult.put(partitionName, status.name());
    }
  }
  return partitionHealthResult;
}