Java Code Examples for org.apache.helix.model.IdealState#getInstanceStateMap()

public Set<TopicPartition> getTopicPartitionBlacklist() {
  Set<TopicPartition> topicPartitionBlacklist = new HashSet<>();
  List<String> topicList = _helixAdmin.getResourcesInCluster(_helixClusterName);
  for (String topic : topicList) {
    IdealState is = _helixAdmin.getResourceIdealState(_helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int i = 0; i < numPartitions; i++) {
      Map<String, String> stateMap = is.getInstanceStateMap(String.valueOf(i));
      if (stateMap != null && stateMap.values().iterator().hasNext() &&
          stateMap.values().iterator().next().equalsIgnoreCase(Constants.HELIX_OFFLINE_STATE)) {
        topicPartitionBlacklist.add(new TopicPartition(topic, i));
      }
    }
  }
  return topicPartitionBlacklist;
}
 

private static JSONObject buildIdealState(String topicName, IdealState idealStateForTopic) {
  JSONObject idealStatePartitionToServerMappingJson = new JSONObject();
  if (idealStateForTopic == null) {
    LOGGER.info("Ideal state for topic " + topicName + " is NULL");
  } else {
    for (String partition : idealStateForTopic.getPartitionSet()) {
      Map<String, String> stateMap = idealStateForTopic.getInstanceStateMap(partition);
      if (stateMap != null) {
        for (String server : stateMap.keySet()) {
          if (!idealStatePartitionToServerMappingJson.containsKey(partition)) {
            idealStatePartitionToServerMappingJson.put(partition, new JSONObject());
          }
          idealStatePartitionToServerMappingJson.getJSONObject(partition).put(server, stateMap.get(server));
        }
      }
    }
  }
  return idealStatePartitionToServerMappingJson;
}
 

@Override
public boolean matches(Object o) {
  if (o == null || !(o instanceof IdealState)) {
    return false;
  }
  IdealState is = (IdealState)o;
  if (is.getRebalanceMode() != IdealState.RebalanceMode.CUSTOMIZED ||
      !is.getReplicas().equals("3") ||
      is.getNumPartitions() != this.numPartitions ||
      !is.getStateModelDefRef().equals("OnlineOffline")) {
    return false;
  }
  for (Map.Entry<Integer, List<String>> entry : partitionHostMap.entrySet()) {
    Map<String, String> stateMap = is.getInstanceStateMap(
            this.resource + "$" + entry.getKey());
    if (stateMap.size() != entry.getValue().size()) {
      return false;
    }
    for (String host : entry.getValue()) {
      if (!(stateMap.containsKey(host) && stateMap.get(host).equals("ONLINE"))) {
        return false;
      }
    }
  }
  return true;
}
 

private boolean hasInstanceMapChanged(Map<Partition, Map<String, String>> newAssiments,
    IdealState idealState) {
  Set<Partition> partitions = new HashSet<Partition>(newAssiments.keySet());
  for (String p : idealState.getPartitionSet()) {
    partitions.add(new Partition(p));
  }

  for (Partition partition : partitions) {
    Map<String, String> instanceMap = newAssiments.get(partition);
    Map<String, String> existInstanceMap =
        idealState.getInstanceStateMap(partition.getPartitionName());
    if (instanceMap == null && existInstanceMap == null) {
      continue;
    }
    if (instanceMap == null || existInstanceMap == null || !instanceMap
        .equals(existInstanceMap)) {
      return true;
    }
  }

  return false;
}
 

private void verifySemiAutoMasterSlaveAssignment(IdealState idealState) {
  for (String partition : idealState.getPartitionSet()) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
    List<String> preferenceList = idealState.getPreferenceList(partition);
    int numMaster = 0;

    for (String ins : preferenceList) {
      Assert.assertTrue(instanceStateMap.containsKey(ins),
          String.format("Instance %s from preference list not in the map", ins));
      String state = instanceStateMap.get(ins);
      Assert.assertTrue(state.equals(MasterSlaveSMD.States.MASTER.name())
          || state.equals(MasterSlaveSMD.States.SLAVE.name()), "Actual State" + state);
      if (state.equals(MasterSlaveSMD.States.MASTER.name())) {
        numMaster++;
      }
    }

    Assert.assertEquals(numMaster, 1);
  }
}
 

private IdealState updateIdealState(IdealState idealState, int newNumReplicas) {
  idealState.setReplicas(Integer.toString(newNumReplicas));
  Set<String> segmentIds = idealState.getPartitionSet();
  for (String segmentId : segmentIds) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId);
    if (instanceStateMap.size() > newNumReplicas) {
      Set<String> keys = instanceStateMap.keySet();
      while (instanceStateMap.size() > newNumReplicas) {
        instanceStateMap.remove(keys.iterator().next());
      }
    } else if (instanceStateMap.size() < newNumReplicas) {
      throw new RuntimeException(
          "Segment " + segmentId + " has " + instanceStateMap.size() + " replicas but want changed to "
              + newNumReplicas);
    }
  }
  return idealState;
}
 

private boolean shouldDeleteInProgressLLCSegment(String segmentName, IdealState idealState,
    RealtimeSegmentZKMetadata realtimeSegmentZKMetadata) {
  if (idealState == null) {
    return false;
  }
  // delete a segment only if it is old enough (5 days) or else,
  // 1. latest segment could get deleted in the middle of repair by RealtimeSegmentValidationManager
  // 2. for a brand new segment, if this code kicks in after new metadata is created but ideal state entry is not yet created (between step 2 and 3),
  // the latest segment metadata could get marked for deletion
  if (System.currentTimeMillis() - realtimeSegmentZKMetadata.getCreationTime()
      <= OLD_LLC_SEGMENTS_RETENTION_IN_MILLIS) {
    return false;
  }
  Map<String, String> stateMap = idealState.getInstanceStateMap(segmentName);
  if (stateMap == null) {
    // Segment is in property store but not in ideal state, delete it
    return true;
  } else {
    // Delete segment if all of its replicas are OFFLINE
    Set<String> states = new HashSet<>(stateMap.values());
    return states.size() == 1 && states.contains(CommonConstants.Helix.StateModel.SegmentStateModel.OFFLINE);
  }
}
 

private void updateIdealstateInfo(
    Map<String, Integer> topicPartitionMapForIdealState,
    IdealState idealStateForTopic) {
  for (String partition : idealStateForTopic.getPartitionSet()) {
    Map<String, String> idealStatesMap = idealStateForTopic.getInstanceStateMap(partition);
    for (String instance : idealStatesMap.keySet()) {
      if (!topicPartitionMapForIdealState.containsKey(instance)) {
        topicPartitionMapForIdealState.put(instance, 1);
      } else {
        topicPartitionMapForIdealState.put(instance,
            topicPartitionMapForIdealState.get(instance) + 1);
      }
    }
  }
}
 

@Override
@Get
public Representation get() {
  JSONObject responseJson = new JSONObject();
  List<TopicAndPartition> noProgressTopicPartition = _helixMirrorMakerManager.getOffsetMonitor().getNoProgessTopicPartitions();
  if (noProgressTopicPartition == null || noProgressTopicPartition.size() == 0) {
    return new StringRepresentation(responseJson.toJSONString());
  }
  JSONArray jsonArray = new JSONArray();
  for (TopicAndPartition info : noProgressTopicPartition) {

    JSONObject node = new JSONObject();
    node.put("topic", info.topic());
    node.put("partition", info.partition());

    IdealState idealStateForTopic =
        _helixMirrorMakerManager.getIdealStateForTopic(info.topic());
    Map<String, String> idealStateMap = idealStateForTopic.getInstanceStateMap(String.valueOf(info.partition()));
    ExternalView externalViewForTopic =
        _helixMirrorMakerManager.getExternalViewForTopic(info.topic());
    Map<String, String> stateMap = externalViewForTopic.getStateMap(String.valueOf(info.partition()));
    if (idealStateMap != null && idealStateMap.keySet().size() != 0) {
      node.put("idealWorker", idealStateMap.keySet().iterator().next());
    }
    if (stateMap != null && stateMap.keySet().size() != 0) {
      node.put("actualWorker", stateMap.keySet().iterator().next());
    }
    jsonArray.add(node);

  }
  responseJson.put("topics", jsonArray);

  return new StringRepresentation(responseJson.toJSONString());
}
 

private boolean assertIdealStateOnce(String topicName, int partition, String expected) {
  IdealState idealStateForTopic =
      helixMirrorMakerManager.getIdealStateForTopic(topicName);
  if (idealStateForTopic.getInstanceStateMap(String.valueOf(partition)) == null || idealStateForTopic.getInstanceStateMap(String.valueOf(partition)).values().size() == 0) {
    Assert.fail(String.format("fail to find IdealState for topic %s, partition %d", topicName, partition));
  }
  String externalState =
      idealStateForTopic.getInstanceStateMap(String.valueOf(partition)).values().iterator().next();
  Assert.assertEquals(externalState, expected, "unexpected idealstate");
  return true;
}
 

private void updateBestPossibleStateOutput(BestPossibleStateOutput output, Resource resource,
    IdealState computedIdealState) {
  output.setPreferenceLists(resource.getResourceName(), computedIdealState.getPreferenceLists());
  for (Partition partition : resource.getPartitions()) {
    Map<String, String> newStateMap =
        computedIdealState.getInstanceStateMap(partition.getPartitionName());
    output.setState(resource.getResourceName(), partition, newStateMap);
  }
}
 

@Override
public ResourceAssignment computeBestPossiblePartitionState(ResourceControllerDataProvider cache,
    IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
  // Looking for cached BestPossible mapping for this resource, if it is already there, do not recompute it again.
  // The cached mapping will be cleared in ResourceControllerDataProvider if there is anything changed in cluster state that can
  // cause the potential changes in BestPossible state.
  ResourceAssignment partitionMapping =
      cache.getCachedResourceAssignment(resource.getResourceName());
  if (partitionMapping != null) {
    return partitionMapping;
  }

  LOG.info("Computing BestPossibleMapping for " + resource.getResourceName());

  String stateModelDefName = idealState.getStateModelDefRef();
  StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
  partitionMapping = new ResourceAssignment(resource.getResourceName());
  for (Partition partition : resource.getPartitions()) {
    Map<String, String> currentStateMap =
        currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
    Set<String> disabledInstancesForPartition =
        cache.getDisabledInstancesForPartition(resource.getResourceName(), partition.toString());
    Map<String, String> idealStateMap =
        idealState.getInstanceStateMap(partition.getPartitionName());
    Map<String, String> bestStateForPartition =
        computeCustomizedBestStateForPartition(cache, stateModelDef, idealStateMap,
            currentStateMap, disabledInstancesForPartition, idealState.isEnabled());
    partitionMapping.addReplicaMap(partition, bestStateForPartition);
  }

  cache.setCachedResourceAssignment(resource.getResourceName(), partitionMapping);
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("Processing resource: %s", resource.getResourceName()));
    LOG.debug(String.format("Final Mapping of resource : %s", partitionMapping.toString()));
  }
  return partitionMapping;
}
 

private void verifyAssignmentInIdealStateWithPersistDisabled(IdealState idealState,
    Set<String> excludedInstances) {
  boolean mapFieldEmpty = true;
  boolean assignmentNotChanged = false;
  for (String partition : idealState.getPartitionSet()) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
    if (instanceStateMap == null || instanceStateMap.isEmpty()) {
      continue;
    }
    mapFieldEmpty = false;
    Set<String> instancesInMap = instanceStateMap.keySet();
    for (String ins : excludedInstances) {
      if (instancesInMap.contains(ins)) {
        // if at least one excluded instance is included, it means assignment was not updated.
        assignmentNotChanged = true;
      }
      if (idealState.getRebalanceMode() == RebalanceMode.FULL_AUTO) {
        List<String> instanceList = idealState.getPreferenceList(partition);
        if (instanceList.contains(ins)) {
          assignmentNotChanged = true;
        }
      }
    }
  }

  Assert.assertTrue((mapFieldEmpty || assignmentNotChanged),
      "BestPossible assignment was updated.");
}
 

/**
 * Validate instances for each partition is on different zone and with necessary tagged
 * instances.
 */
private void validateNoZeroReplica(IdealState is, ExternalView ev) {
  int replica = is.getReplicaCount(NUM_NODE);
  StateModelDefinition stateModelDef =
      BuiltInStateModelDefinitions.valueOf(is.getStateModelDefRef()).getStateModelDefinition();

  for (String partition : is.getPartitionSet()) {
    Map<String, String> evStateMap = ev.getRecord().getMapField(partition);
    Map<String, String> isStateMap = is.getInstanceStateMap(partition);
    validateMap(is.getResourceName(), partition, replica, evStateMap, stateModelDef);
    validateMap(is.getResourceName(), partition, replica, isStateMap, stateModelDef);
  }
}
 

/**
 * Check instance is already in the stable state. Here stable means all the ideal state mapping
 * matches external view (view of current state).
 * It requires PERSIST_INTERMEDIATE_ASSIGNMENT turned on!
 * @param dataAccessor
 * @param instanceName
 * @return
 */
public static boolean isInstanceStable(HelixDataAccessor dataAccessor, String instanceName) {
  PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
  ClusterConfig clusterConfig = dataAccessor.getProperty(keyBuilder.clusterConfig());
  if (clusterConfig == null) {
    throw new HelixException("Missing cluster config!");
  }
  if (!clusterConfig.isPersistIntermediateAssignment()) {
    throw new HelixException("isInstanceStable needs persist assignment on!");
  }

  List<String> idealStateNames = dataAccessor.getChildNames(keyBuilder.idealStates());
  for (String idealStateName : idealStateNames) {
    IdealState idealState = dataAccessor.getProperty(keyBuilder.idealStates(idealStateName));
    if (idealState == null || !idealState.isEnabled() || !idealState.isValid()
        || TaskConstants.STATE_MODEL_NAME.equals(idealState.getStateModelDefRef())) {
      continue;
    }

    ExternalView externalView = dataAccessor.getProperty(keyBuilder.externalView(idealStateName));
    if (externalView == null) {
      throw new HelixException(
          String.format("Resource %s does not have external view!", idealStateName));
    }
    for (String partition : idealState.getPartitionSet()) {
      Map<String, String> isPartitionMap = idealState.getInstanceStateMap(partition);
      if (isPartitionMap == null) {
        throw new HelixException(String
            .format("Partition %s of resource %s does not have an ideal state partition map",
                partition, idealStateName));
      }
      if (isPartitionMap.containsKey(instanceName)) {
        Map<String, String> evPartitionMap = externalView.getStateMap(partition);
        if (evPartitionMap == null) {
          throw new HelixException(String
              .format("Partition %s of resource %s does not have an external view partition map",
                  partition, idealStateName));
        }
        if (!evPartitionMap.containsKey(instanceName)
            || !evPartitionMap.get(instanceName).equals(isPartitionMap.get(instanceName))) {
          // only checks the state from IS matches EV. Return false when
          // 1. This partition not has current state on this instance
          // 2. The state does not match the state on ideal state
          return false;
        }
      }
    }
  }
  return true;
}