Java Code Examples for org.apache.helix.model.IdealState#getNumPartitions()

public static IdealState resetCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String partitionToReplace, String newInstanceName) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(oldNumPartitions);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
    String instanceToUse = partitionName.equals(partitionToReplace) ? newInstanceName : instanceName;
    customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceToUse, "ONLINE");
  }

  return customModeIdealStateBuilder.build();
}
 

public static IdealState resetCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String oldPartition, String newPartition, String newInstanceName) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(oldNumPartitions);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
    String instanceToUse = partitionName.equals(oldPartition) ? newInstanceName : instanceName;
    String partitionToUse = partitionName.equals(oldPartition) ? newPartition : oldPartition;
    customModeIdealStateBuilder.assignInstanceAndState(partitionToUse, instanceToUse, "ONLINE");
  }

  return customModeIdealStateBuilder.build();
}
 

public static IdealState resetCustomIdealStateFor(IdealState oldIdealState,
    String topicName, List<String> instanceToReplace, String availableInstance, int maxNumReplica) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions).setNumReplica(maxNumReplica)
      .setMaxPartitionsPerNode(oldNumPartitions);
  for (String partitionName : oldIdealState.getPartitionSet()) {
    for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) {
      String instanceToUse = instanceToReplace.contains(instanceName) ? availableInstance : instanceName;
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceToUse, "ONLINE");
      if (instanceToReplace.contains(instanceName)) {
        LOGGER.info("replaceing: route: {}@{}, old {}, new {}",
            topicName, partitionName, instanceName, instanceToUse);
      }
    }
  }

  return customModeIdealStateBuilder.build();
}
 

public static IdealState expandCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String newPartition, InstanceTopicPartitionHolder instance) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions + 1).setNumReplica(1)
      .setMaxPartitionsPerNode(oldNumPartitions + 1);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
    customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
  }

  customModeIdealStateBuilder.assignInstanceAndState(newPartition, instance.getInstanceName(), "ONLINE");

  return customModeIdealStateBuilder.build();
}
 

public static IdealState expandCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String newPartition, List<String> instances, int maxNumReplica) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions + 1).setNumReplica(maxNumReplica)
      .setMaxPartitionsPerNode(oldNumPartitions + 1);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) {
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
    }
  }

  for (String instance : instances) {
    customModeIdealStateBuilder.assignInstanceAndState(newPartition, instance, "ONLINE");
  }

  return customModeIdealStateBuilder.build();
}
 

public static IdealState expandInstanceCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String newPartition, List<String> instances, int maxNumReplica) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions).setNumReplica(maxNumReplica)
      .setMaxPartitionsPerNode(oldNumPartitions);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    for (String instanceName : oldIdealState.getInstanceStateMap(partitionName).keySet()) {
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
    }
    if (partitionName.equals(newPartition)) {
      for (String newInstanceName : instances) {
        customModeIdealStateBuilder.assignInstanceAndState(partitionName, newInstanceName, "ONLINE");
      }
    }
  }

  return customModeIdealStateBuilder.build();
}
 

public static IdealState shrinkCustomIdealStateFor(IdealState oldIdealState,
    String topicName, String partitionToDelete) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  int oldNumPartitions = oldIdealState.getNumPartitions();

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(oldNumPartitions - 1).setNumReplica(1)
      .setMaxPartitionsPerNode(oldNumPartitions - 1);

  for (String partitionName : oldIdealState.getPartitionSet()) {
    if (!partitionName.equals(partitionToDelete)) {
      String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
    }
  }

  return customModeIdealStateBuilder.build();
}
 

public Set<TopicPartition> getTopicPartitionBlacklist() {
  Set<TopicPartition> topicPartitionBlacklist = new HashSet<>();
  List<String> topicList = _helixAdmin.getResourcesInCluster(_helixClusterName);
  for (String topic : topicList) {
    IdealState is = _helixAdmin.getResourceIdealState(_helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int i = 0; i < numPartitions; i++) {
      Map<String, String> stateMap = is.getInstanceStateMap(String.valueOf(i));
      if (stateMap != null && stateMap.values().iterator().hasNext() &&
          stateMap.values().iterator().next().equalsIgnoreCase(Constants.HELIX_OFFLINE_STATE)) {
        topicPartitionBlacklist.add(new TopicPartition(topic, i));
      }
    }
  }
  return topicPartitionBlacklist;
}
 

public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) {
  Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int partition = 0; partition < numPartitions; ++partition) {
      if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) {
        TopicPartition tpi = new TopicPartition(topic, partition);
        unassignedPartitions.add(tpi);
      }
    }
  }
  return unassignedPartitions;
}
 

private Map<String, Integer> getTopicsPartitions(List<String> topics) {
  Map<String, Integer> topicsPartitions = new HashMap<>();
  for (String topic : topics) {
    IdealState idealState = _helixMirrorMakerManager.getIdealStateForTopic(topic);
    // TODO: make it compatible with controller
    if (idealState != null) {
      if (isController) {
        int partitions = idealState.getNumPartitions();
        if (partitions > 0) {
          topicsPartitions.put(topic, partitions);
        }
      } else {
        Iterator<String> iter = idealState.getPartitionSet().iterator();
        while (iter.hasNext()) {
          String route = iter.next();
          if (route.substring(1).startsWith(_simpleSrcKafkaCluster)) {
            topicsPartitions.put(topic, idealState.getNumPartitions());
            break;
          }
        }
      }
    }
  }
  return topicsPartitions;
}
 

@Override
public boolean matches(Object o) {
  if (o == null || !(o instanceof IdealState)) {
    return false;
  }
  IdealState is = (IdealState)o;
  if (is.getRebalanceMode() != IdealState.RebalanceMode.CUSTOMIZED ||
      !is.getReplicas().equals("3") ||
      is.getNumPartitions() != this.numPartitions ||
      !is.getStateModelDefRef().equals("OnlineOffline")) {
    return false;
  }
  for (Map.Entry<Integer, List<String>> entry : partitionHostMap.entrySet()) {
    Map<String, String> stateMap = is.getInstanceStateMap(
            this.resource + "$" + entry.getKey());
    if (stateMap.size() != entry.getValue().size()) {
      return false;
    }
    for (String host : entry.getValue()) {
      if (!(stateMap.containsKey(host) && stateMap.get(host).equals("ONLINE"))) {
        return false;
      }
    }
  }
  return true;
}
 

public static PistachioClusterInfo getClusterInfo() {
    try {
        String zookeeperConnStr = ConfigurationManager.getConfiguration().getString("Pistachio.ZooKeeper.Server");
        ZKHelixAdmin admin = new ZKHelixAdmin(zookeeperConnStr);
        IdealState idealState = admin.getResourceIdealState("PistachiosCluster", "PistachiosResource");
        PistachioClusterInfo info = new PistachioClusterInfo();
        info.numPartitions = idealState.getNumPartitions();
        info.numReplicas = Integer.parseInt(idealState.getReplicas());
        info.hostList = admin.getInstancesInCluster("PistachiosCluster");

        logger.info("num partitions: {}, num Replicas: {}, hostList: {}.", info.numPartitions,
            info.numReplicas, Joiner.on(",").join(info.hostList.toArray()));

        return info;
    } catch (Exception e) {
        logger.info("error getting cluster info", e);
        return null;
    }
}
 

@Test
public void testEnableDisablePartitions() throws InterruptedException {
  _admin.enablePartition(false, CLUSTER_NAME, (PARTICIPANT_PREFIX + "_" + _startPort),
      WorkflowGenerator.DEFAULT_TGT_DB, Arrays.asList(new String[] { "TestDB_0", "TestDB_2" }));
  _admin.enablePartition(false, CLUSTER_NAME, (PARTICIPANT_PREFIX + "_" + (_startPort + 1)),
      WorkflowGenerator.DEFAULT_TGT_DB, Arrays.asList(new String[] { "TestDB_0", "TestDB_2" }));

  IdealState idealState =
      _admin.getResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  List<String> preferenceList =
      Arrays.asList(new String[] { "localhost_12919", "localhost_12918" });
  for (String partitionName : idealState.getPartitionSet()) {
    idealState.setPreferenceList(partitionName, preferenceList);
  }
  idealState.setRebalanceMode(IdealState.RebalanceMode.SEMI_AUTO);
  _admin.setResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, idealState);

  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  String workflowName = TestHelper.getTestMethodName();
  Workflow.Builder builder = new Workflow.Builder(workflowName);
  JobConfig.Builder jobBuilder =
      new JobConfig.Builder().setWorkflow(workflowName).setCommand(MockTask.TASK_COMMAND)
          .setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB)
          .setTargetPartitionStates(Collections.singleton("SLAVE"));
  builder.addJob("JOB", jobBuilder);
  _driver.start(builder.build());
  Thread.sleep(2000L);
  JobContext jobContext =
      _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "JOB"));
  int n = idealState.getNumPartitions();
  for ( int i = 0; i < n; i++) {
    String targetPartition = jobContext.getTargetForPartition(i);
    if (targetPartition.equals("TestDB_0") || targetPartition.equals("TestDB_2")) {
      Assert.assertEquals(jobContext.getPartitionState(i), null);
    } else {
      Assert.assertEquals(jobContext.getPartitionState(i), TaskPartitionState.COMPLETED);
    }
  }
}
 

public synchronized void deleteTopicInMirrorMaker(String topicName, String src, String dst,
    String pipeline)
    throws Exception {
  _lock.lock();
  try {
    LOGGER.info("Trying to delete topic: {} in pipeline: {}", topicName, pipeline);

    InstanceTopicPartitionHolder instance = _topicToPipelineInstanceMap.get(topicName)
        .get(pipeline);
    IdealState currIdealState = getIdealStateForTopic(topicName);
    if (currIdealState.getPartitionSet().contains(instance.getRouteString())
        && currIdealState.getNumPartitions() == 1) {
      _helixAdmin.dropResource(_helixClusterName, topicName);
    } else {
      _helixAdmin.setResourceIdealState(_helixClusterName, topicName,
          IdealStateBuilder
              .shrinkCustomIdealStateFor(currIdealState, topicName, instance.getRouteString()));
    }
    TopicPartition tp = _kafkaValidationManager.getClusterToObserverMap().get(src)
        .getTopicPartitionWithRefresh(topicName);
    instance.removeTopicPartition(tp);
    _topicToPipelineInstanceMap.get(topicName).remove(pipeline);
    if (_topicToPipelineInstanceMap.get(topicName).keySet().size() == 0) {
      _topicToPipelineInstanceMap.remove(topicName);
    }
    if (instance.getServingTopicPartitionSet().isEmpty()) {
      _availableControllerList.add(instance.getInstanceName());
      _assignedControllerCount.dec();
    }
  } finally {
    _lock.unlock();
  }
}
 

public long getTotalPartition(String resource) {
    if (totalParition == -1) {
        synchronized(totalParition) {
            if (totalParition == -1) {
                ZKHelixAdmin admin = new ZKHelixAdmin(zkAddress);
                IdealState idealState = admin.getResourceIdealState(helixClusterName, resource);
                totalParition = (long)idealState.getNumPartitions();
            }
        }
    }

    return totalParition;
}
 

public synchronized void updateTopicPartitionStateInMirrorMaker(String topicName, int partition,
    String state) {
  updateCurrentServingInstance();
  if (!Constants.HELIX_OFFLINE_STATE.equalsIgnoreCase(state) && !Constants.HELIX_ONLINE_STATE
      .equalsIgnoreCase(state)) {
    throw new IllegalArgumentException(
        String.format("Failed to update topic %s, partition %d to invalid state %s.",
            topicName, partition, state));
  }

  IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, topicName);
  String partitionName = String.valueOf(partition);
  if (idealState == null ||
      partition >= idealState.getNumPartitions()) {
    throw new IllegalArgumentException(
        String.format("Topic %s, partition %d not exists in current route.",
            topicName, partition));
  }

  String instanceName;
  if (idealState.getInstanceStateMap(partitionName).keySet().isEmpty()) {
    if (Constants.HELIX_OFFLINE_STATE.equalsIgnoreCase(state)) {
      throw new IllegalArgumentException(
          String.format("Topic %s, partition %d not exists in current route.",
              topicName, partition));
    } else if (_currentServingInstance.isEmpty()) {
      throw new InternalError("No available worker");
    }
    instanceName = _currentServingInstance.poll().getInstanceName();
  } else {
    instanceName = idealState.getInstanceStateMap(partitionName).keySet().iterator().next();
    String oldState = idealState.getInstanceStateMap(partitionName).get(instanceName);
    if (oldState.equalsIgnoreCase(state)) {
      throw new IllegalArgumentException(String.format("Topic %s, partition %d already set %s",
          idealState.getResourceName(), partition, state));
    }
  }

  idealState.setPartitionState(partitionName, instanceName, state);
  _helixAdmin.setResourceIdealState(_helixClusterName, topicName, idealState);
}
 

public boolean init() {

        boolean initialized = false;

        logger.info("Initializing profile server...........");
        logger.info("do nothing setting {}", doNothing);
        try {
            // open profile store
            Configuration conf = ConfigurationManager.getConfiguration();
        ZKHelixAdmin admin = new ZKHelixAdmin(conf.getString(ZOOKEEPER_SERVER));
        IdealState idealState = admin.getResourceIdealState("PistachiosCluster", "PistachiosResource");
        long totalParition = (long)idealState.getNumPartitions();
            profileStore = new LocalStorageEngine(
                    conf.getString(PROFILE_BASE_DIR),
                    (int)totalParition, 8,
                    conf.getInt("StorageEngine.KC.RecordsPerPartition"),
                    conf.getLong("StorageEngine.KC.MemoryPerPartition"));
            ProcessorRegistry.getInstance().init();
            logger.info("creating helix partition sepctator {} {} {}", conf.getString(ZOOKEEPER_SERVER, "EMPTY"),
                    "PistachiosCluster", conf.getString(PROFILE_HELIX_INSTANCE_ID, "EMPTY"));
            helixPartitionSpectator = HelixPartitionSpectator.getInstance(
                    conf.getString(ZOOKEEPER_SERVER), // zkAddr
                    "PistachiosCluster",
                    NativeUtils.getHostname() //InetAddress.getLocalHost().getHostName() //conf.getString(PROFILE_HELIX_INSTANCE_ID) // instanceName
                    );
            // Partition Manager for line spending
            manager = new HelixPartitionManager<>(
                    conf.getString(ZOOKEEPER_SERVER), // zkAddr
                    "PistachiosCluster",
                    NativeUtils.getHostname() //InetAddress.getLocalHost().getHostName() //conf.getString(PROFILE_HELIX_INSTANCE_ID) // instanceName
                    );
            //manager.start("BootstrapOnlineOffline", new BootstrapOnlineOfflineStateModelFactory(new StorePartitionHandlerFactory()));
            manager.start("MasterSlave", new BootstrapOnlineOfflineStateModelFactory(new StorePartitionHandlerFactory()));
            //            }

            initialized = true;
        } catch (Exception e) {
            logger.error("Failed to initialize ProfileServerModule", e);
        }
        logger.info("Finished initializing profile server...........");

        return initialized;
    }
 

void rebalance(String clusterName, String resourceName, int replica, String keyPrefix,
    List<String> instanceNames, String groupId) {
  logger.info("Rebalance resource {} with replica {} in cluster {}.", resourceName, replica,
      clusterName);
  // ensure we get the same idealState with the same set of instances
  Collections.sort(instanceNames);

  IdealState idealState = getResourceIdealState(clusterName, resourceName);
  if (idealState == null) {
    throw new HelixException("Resource: " + resourceName + " has NOT been added yet");
  }

  if (groupId != null && groupId.length() > 0) {
    idealState.setInstanceGroupTag(groupId);
  }
  idealState.setReplicas(Integer.toString(replica));
  int partitions = idealState.getNumPartitions();
  String stateModelName = idealState.getStateModelDefRef();
  StateModelDefinition stateModDef = getStateModelDef(clusterName, stateModelName);

  if (stateModDef == null) {
    throw new HelixException("cannot find state model: " + stateModelName);
  }
  // StateModelDefinition def = new StateModelDefinition(stateModDef);

  List<String> statePriorityList = stateModDef.getStatesPriorityList();

  String masterStateValue = null;
  String slaveStateValue = null;
  replica--;

  for (String state : statePriorityList) {
    String count = stateModDef.getNumInstancesPerState(state);
    if (count.equals("1")) {
      if (masterStateValue != null) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      masterStateValue = state;
    } else if (count.equalsIgnoreCase("R")) {
      if (slaveStateValue != null) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      slaveStateValue = state;
    } else if (count.equalsIgnoreCase("N")) {
      if (!(masterStateValue == null && slaveStateValue == null)) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      replica = instanceNames.size() - 1;
      masterStateValue = slaveStateValue = state;
    }
  }
  if (masterStateValue == null && slaveStateValue == null) {
    throw new HelixException("Invalid or unsupported state model definition");
  }

  if (masterStateValue == null) {
    masterStateValue = slaveStateValue;
  }
  if (idealState.getRebalanceMode() != RebalanceMode.FULL_AUTO
      && idealState.getRebalanceMode() != RebalanceMode.USER_DEFINED) {
    ZNRecord newIdealState = DefaultIdealStateCalculator
        .calculateIdealState(instanceNames, partitions, replica, keyPrefix, masterStateValue,
            slaveStateValue);

    // for now keep mapField in SEMI_AUTO mode and remove listField in CUSTOMIZED mode
    if (idealState.getRebalanceMode() == RebalanceMode.SEMI_AUTO) {
      idealState.getRecord().setListFields(newIdealState.getListFields());
      // TODO: need consider to remove this.
      idealState.getRecord().setMapFields(newIdealState.getMapFields());
    }
    if (idealState.getRebalanceMode() == RebalanceMode.CUSTOMIZED) {
      idealState.getRecord().setMapFields(newIdealState.getMapFields());
    }
  } else {
    for (int i = 0; i < partitions; i++) {
      String partitionName = keyPrefix + "_" + i;
      idealState.getRecord().setMapField(partitionName, new HashMap<String, String>());
      idealState.getRecord().setListField(partitionName, new ArrayList<String>());
    }
  }
  setResourceIdealState(clusterName, resourceName, idealState);
}
 

/**
 * Validates ResourceConfig's weight field against the given ClusterConfig.
 * @param clusterConfig
 * @param resourceConfig
 * @param idealState
 * @return true if ResourceConfig has all the required fields. False otherwise.
 */
private boolean validateWeightForResourceConfig(ClusterConfig clusterConfig,
    ResourceConfig resourceConfig, IdealState idealState) {
  if (resourceConfig == null) {
    if (clusterConfig.getDefaultPartitionWeightMap().isEmpty()) {
      logger.error(
          "ResourceConfig for {} is null, and there are no default weights set in ClusterConfig!",
          idealState.getResourceName());
      return false;
    }
    // If ResourceConfig is null AND the default partition weight map is defined, and the map has all the required keys, we consider this valid since the default weights will be used
    // Need to check the map contains all the required keys
    if (clusterConfig.getDefaultPartitionWeightMap().keySet()
        .containsAll(clusterConfig.getInstanceCapacityKeys())) {
      // Contains all the required keys, so consider it valid since it will use the default weights
      return true;
    }
    logger.error(
        "ResourceConfig for {} is null, and ClusterConfig's default partition weight map doesn't have all the required keys!",
        idealState.getResourceName());
    return false;
  }

  // Parse the entire capacityMap from ResourceConfig
  Map<String, Map<String, Integer>> capacityMap;
  try {
    capacityMap = resourceConfig.getPartitionCapacityMap();
  } catch (IOException ex) {
    logger.error("Invalid partition capacity configuration of resource: {}",
        idealState.getResourceName(), ex);
    return false;
  }

  Set<String> capacityMapSet = new HashSet<>(capacityMap.keySet());
  boolean hasDefaultCapacity = capacityMapSet.contains(ResourceConfig.DEFAULT_PARTITION_KEY);
  // Remove DEFAULT key
  capacityMapSet.remove(ResourceConfig.DEFAULT_PARTITION_KEY);

  // Make sure capacityMap contains all partitions defined in IdealState
  // Here, IdealState has not been rebalanced, so listFields might be null, in which case, we would get an emptyList from getPartitionSet()
  // So check using numPartitions instead
  // This check allows us to fail early on instead of having to loop through all partitions
  if (capacityMapSet.size() != idealState.getNumPartitions() && !hasDefaultCapacity) {
    logger.error(
        "ResourceConfig for {} does not have all partitions defined in PartitionCapacityMap!",
        idealState.getResourceName());
    return false;
  }

  // Loop through all partitions and validate
  capacityMap.keySet().forEach(partitionName -> WagedValidationUtil
      .validateAndGetPartitionCapacity(partitionName, resourceConfig, capacityMap,
          clusterConfig));
  return true;
}
 

public synchronized String validateExternalView() {
  try {
    Map<String, Integer> topicPartitionMapForIdealState =
        new HashMap<String, Integer>();
    Map<String, Integer> topicPartitionMapForExternalView =
        new HashMap<String, Integer>();
    int numOnlineTopicPartitions = 0;
    int numOfflineTopicPartitions = 0;
    int numErrorTopicPartitions = 0;
    int numTopicPartitions = 0;
    int numServingTopics = 0;
    int numErrorTopics = 0;
    for (String topicName : _helixMirrorMakerManager.getTopicLists()) {
      numServingTopics++;
      IdealState idealStateForTopic =
          _helixMirrorMakerManager.getIdealStateForTopic(topicName);
      ExternalView externalViewForTopic =
          _helixMirrorMakerManager.getExternalViewForTopic(topicName);
      numTopicPartitions += idealStateForTopic.getNumPartitions();
      if (idealStateForTopic.getNumPartitions() != externalViewForTopic.getPartitionSet()
          .size()) {
        numErrorTopics++;
        LOGGER.error(
            "For topic:{}, number of partitions for IdealState: {} doesn't match ExternalView: {}",
            topicName, idealStateForTopic, externalViewForTopic);
      }

      // IdealState Counting
      updateIdealstateInfo(topicPartitionMapForIdealState, idealStateForTopic);
      // ExternalView Counting
      for (String partition : externalViewForTopic.getPartitionSet()) {
        Map<String, String> stateMap = externalViewForTopic.getStateMap(partition);
        for (String instance : stateMap.keySet()) {
          String state = stateMap.get(instance);
          if (!topicPartitionMapForExternalView.containsKey(instance)) {
            topicPartitionMapForExternalView.put(instance, 1);
          } else {
            topicPartitionMapForExternalView.put(instance,
                topicPartitionMapForExternalView.get(instance) + 1);
          }
          if ("ONLINE".equalsIgnoreCase(state)) {
            numOnlineTopicPartitions++;
          } else if ("OFFLINE".equalsIgnoreCase(state)) {
            numOfflineTopicPartitions++;
          } else if ("ERROR".equalsIgnoreCase(state)) {
            numErrorTopicPartitions++;
          }
        }
      }
    }

    if (_helixMirrorMakerManager.isLeader()) {
      updateMetrics(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions,
          numTopicPartitions, numServingTopics, numErrorTopics);
      updatePerWorkerISMetrics(topicPartitionMapForIdealState);
      updatePerWorkerEVMetrics(topicPartitionMapForExternalView);
    }
    JSONObject perWorkerISCounterJson =
        constructPerWorkerISCounterJson(topicPartitionMapForIdealState);
    JSONObject perWorkerEVCounterJson =
        constructPerWorkerEVCounterJson(topicPartitionMapForExternalView);
    JSONObject validationResultJson = constructValidationResultJson(numOnlineTopicPartitions,
        numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions,
        numServingTopics, numErrorTopics, perWorkerISCounterJson, perWorkerEVCounterJson);
    return validationResultJson.toJSONString();
  } catch (Exception e) {
    JSONObject jsonObject = new JSONObject();
    jsonObject.put("excpetion", e);
    return jsonObject.toJSONString();
  }
}