Java Code Examples for org.apache.helix.model.LiveInstance#getInstanceName()

@Override
public void process(ClusterEvent event) throws Exception {
  HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
  BaseControllerDataProvider cache = event.getAttribute(AttributeName.ControllerDataProvider.name());
  if (manager == null || cache == null) {
    throw new StageException("Missing attributes in event:" + event
        + ". Requires HelixManager | DataCache");
  }

  HelixManagerProperties properties = manager.getProperties();
  Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
  for (LiveInstance liveInstance : liveInstanceMap.values()) {
    String participantVersion = liveInstance.getHelixVersion();
    if (!properties.isParticipantCompatible(participantVersion)) {
      String errorMsg =
          "incompatible participant. pipeline will not continue. " + "controller: "
              + manager.getInstanceName() + ", controllerVersion: " + properties.getVersion()
              + ", minimumSupportedParticipantVersion: "
              + properties.getProperty("minimum_supported_version.participant")
              + ", participant: " + liveInstance.getInstanceName() + ", participantVersion: "
              + participantVersion;
      LogUtil.logError(LOG, event.getEventId(), errorMsg);
      throw new StageException(errorMsg);
    }
  }
}
 

/**
 * Gets Helix leader in the cluster. Null if there is no leader.
 * @param helixManager helix manager
 * @return instance id of Helix cluster leader, e.g. localhost_9000.
 */
public static String getHelixClusterLeader(HelixManager helixManager) {
  HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor();
  PropertyKey propertyKey = helixDataAccessor.keyBuilder().controllerLeader();
  LiveInstance liveInstance = helixDataAccessor.getProperty(propertyKey);
  if (liveInstance == null) {
    LOGGER.warn("Helix leader ZNode is missing");
    return null;
  }
  String helixLeaderInstanceId = liveInstance.getInstanceName();
  String helixVersion = liveInstance.getHelixVersion();
  long modifiedTime = liveInstance.getModifiedTime();
  LOGGER.info("Getting Helix leader: {}, Helix version: {}, mtime: {}", helixLeaderInstanceId, helixVersion,
      modifiedTime);
  return helixLeaderInstanceId;
}
 

private void refresh(Map<String, Map<String, Map<String, CurrentState>>> currentStateMap) {
  Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();
  if (currentStateMap != null && !currentStateMap.isEmpty()) {
    for (InstanceConfig config : _instanceConfigs) {
      instanceConfigMap.put(config.getId(), config);
    }
    for (LiveInstance liveInstance : _liveInstances) {
      String instanceName = liveInstance.getInstanceName();
      String sessionId = liveInstance.getEphemeralOwner();
      InstanceConfig instanceConfig = instanceConfigMap.get(instanceName);
      if (instanceConfig == null) {
        logger.warn(
            "Participant {} is not found with proper configuration information. It might already be removed from the cluster. "
                + "Skip recording partition assignments that are related to this instance.",
            instanceName);
        continue;
      }

      Map<String, CurrentState> currentStates = Collections.emptyMap();
      if (currentStateMap.containsKey(instanceName)
          && currentStateMap.get(instanceName).containsKey(sessionId)) {
        currentStates = currentStateMap.get(instanceName).get(sessionId);
      }

      for (CurrentState currentState : currentStates.values()) {
        String resourceName = currentState.getResourceName();
        Map<String, String> stateMap = currentState.getPartitionStateMap();

        for (String partitionName : stateMap.keySet()) {
          String state = stateMap.get(partitionName);
          addEntry(resourceName, partitionName, state, instanceConfig);
        }
      }
    }
  }
}
 

@Override
public boolean isLeader() {
  String warnLogPrefix = String
      .format("Instance %s is not leader of cluster %s due to", _instanceName, _clusterName);
  if (_instanceType != InstanceType.CONTROLLER
      && _instanceType != InstanceType.CONTROLLER_PARTICIPANT) {
    LOG.warn(String
        .format("%s instance type %s does not match to CONTROLLER/CONTROLLER_PARTICIPANT",
            warnLogPrefix, _instanceType.name()));
    return false;
  }

  if (!isConnected()) {
    LOG.warn(String.format("%s HelixManager is not connected", warnLogPrefix));
    return false;
  }

  try {
    LiveInstance leader = _dataAccessor.getProperty(_keyBuilder.controllerLeader());
    if (leader != null) {
      String leaderName = leader.getInstanceName();
      String sessionId = leader.getEphemeralOwner();
      if (leaderName != null && leaderName.equals(_instanceName) && sessionId
          .equals(_sessionId)) {
        return true;
      }
      LOG.warn(String
          .format("%s current session %s does not match leader session %s", warnLogPrefix,
              _sessionId, sessionId));
    } else {
      LOG.warn(String.format("%s leader ZNode is null", warnLogPrefix));
    }
  } catch (Exception e) {
    LOG.warn(String.format("%s exception happen when session check", warnLogPrefix), e);
  }
  return false;
}
 

@Override
public void process(ClusterEvent event) throws Exception {
  _eventId = event.getEventId();
  ResourceControllerDataProvider cache =
      event.getAttribute(AttributeName.ControllerDataProvider.name());
  final Map<String, Resource> resourceMap =
      event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
  Set<String> aggregationEnabledTypes = cache.getAggregationEnabledCustomizedStateTypes();

  if (cache == null || resourceMap == null) {
    throw new StageException(
        "Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
  }

  Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
  final CustomizedStateOutput customizedStateOutput = new CustomizedStateOutput();

  for (LiveInstance instance : liveInstances.values()) {
    String instanceName = instance.getInstanceName();
    // update customized states.
    for (String customizedStateType : aggregationEnabledTypes) {
      Map<String, CustomizedState> customizedStateMap =
          cache.getCustomizedState(instanceName, customizedStateType);
      updateCustomizedStates(instanceName, customizedStateType, customizedStateMap,
          customizedStateOutput, resourceMap);
    }
  }
  event.addAttribute(AttributeName.CUSTOMIZED_STATE.name(), customizedStateOutput);
}
 

protected String getCurrentLeader(HelixZkClient zkClient, String clusterName) {
  ZKHelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(zkClient));
  Builder keyBuilder = accessor.keyBuilder();

  LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
  if (leader == null) {
    return null;
  }
  return leader.getInstanceName();
}
 

@Override
public void process(ClusterEvent event) throws Exception {
  _eventId = event.getEventId();
  BaseControllerDataProvider cache = event.getAttribute(AttributeName.ControllerDataProvider.name());
  final Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
  final Map<String, Resource> resourceToRebalance =
      event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());

  if (cache == null || resourceMap == null) {
    throw new StageException("Missing attributes in event:" + event
        + ". Requires DataCache|RESOURCE");
  }

  Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
  final CurrentStateOutput currentStateOutput = new CurrentStateOutput();

  for (LiveInstance instance : liveInstances.values()) {
    String instanceName = instance.getInstanceName();
    String instanceSessionId = instance.getEphemeralOwner();

    // update pending messages
    Map<String, Message> messages = cache.getMessages(instanceName);
    Map<String, Message> relayMessages = cache.getRelayMessages(instanceName);
    updatePendingMessages(instance, messages.values(), currentStateOutput, relayMessages.values(), resourceMap);

    // update current states.
    Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName,
        instanceSessionId);
    updateCurrentStates(instance, currentStateMap.values(), currentStateOutput, resourceMap);
  }
  event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);

  final ClusterStatusMonitor clusterStatusMonitor =
      event.getAttribute(AttributeName.clusterStatusMonitor.name());
  if (clusterStatusMonitor != null && cache instanceof ResourceControllerDataProvider) {
    final ResourceControllerDataProvider dataProvider = (ResourceControllerDataProvider) cache;
    reportInstanceCapacityMetrics(clusterStatusMonitor, dataProvider, resourceToRebalance,
        currentStateOutput);
    reportResourcePartitionCapacityMetrics(dataProvider.getAsyncTasksThreadPool(),
        clusterStatusMonitor, dataProvider.getResourceConfigMap().values());
  }
}
 

private void updateCurrentStates(LiveInstance instance, Collection<CurrentState> currentStates,
    CurrentStateOutput currentStateOutput, Map<String, Resource> resourceMap) {
  String instanceName = instance.getInstanceName();
  String instanceSessionId = instance.getEphemeralOwner();

  for (CurrentState currentState : currentStates) {
    if (!instanceSessionId.equals(currentState.getSessionId())) {
      continue;
    }
    String resourceName = currentState.getResourceName();
    String stateModelDefName = currentState.getStateModelDefRef();
    Resource resource = resourceMap.get(resourceName);
    if (resource == null) {
      continue;
    }
    if (stateModelDefName != null) {
      currentStateOutput.setResourceStateModelDef(resourceName, stateModelDefName);
    }

    currentStateOutput.setBucketSize(resourceName, currentState.getBucketSize());

    Map<String, String> partitionStateMap = currentState.getPartitionStateMap();
    for (String partitionName : partitionStateMap.keySet()) {
      Partition partition = resource.getPartition(partitionName);
      if (partition != null) {
        currentStateOutput.setCurrentState(resourceName, partition, instanceName,
            currentState.getState(partitionName));
        currentStateOutput.setEndTime(resourceName, partition, instanceName,
            currentState.getEndTime(partitionName));
        String info = currentState.getInfo(partitionName);
        // This is to avoid null value entries in the map, and reduce memory usage by avoiding extra empty entries in the map.
        if (info != null) {
          currentStateOutput.setInfo(resourceName, partition, instanceName, info);
        }
        String requestState = currentState.getRequestedState(partitionName);
        if (requestState != null) {
          currentStateOutput
              .setRequestedState(resourceName, partition, instanceName, requestState);
        }
      }
    }
  }
}
 

/**
 * Tests the duality of two AssignableInstanceManager instances to model the
 * situation where there is a Controller switch and AssignableInstanceManager is
 * built back from scratch.
 * @throws InterruptedException
 */
@Test
public void testControllerSwitch() throws InterruptedException {
  setupAndRunJobs();

  Map<String, LiveInstance> liveInstanceMap = new HashMap<>();
  Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();

  RoutingTableProvider routingTableProvider = new RoutingTableProvider(_manager);
  Collection<LiveInstance> liveInstances = routingTableProvider.getLiveInstances();
  for (LiveInstance liveInstance : liveInstances) {
    String instanceName = liveInstance.getInstanceName();
    liveInstanceMap.put(instanceName, liveInstance);
    instanceConfigMap.put(instanceName,
        _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instanceName));
  }

  // Get ClusterConfig
  ClusterConfig clusterConfig = _manager.getConfigAccessor().getClusterConfig(CLUSTER_NAME);

  // Initialize TaskDataCache
  HelixDataAccessor accessor = _manager.getHelixDataAccessor();
  TaskDataCache taskDataCache = new TaskDataCache(CLUSTER_NAME);
  Map<String, ResourceConfig> resourceConfigMap =
      accessor.getChildValuesMap(accessor.keyBuilder().resourceConfigs(), true);

  // Wait for the job pipeline
  Thread.sleep(1000);
  taskDataCache.refresh(accessor, resourceConfigMap);

  // Create prev manager and build
  AssignableInstanceManager prevAssignableInstanceManager = new AssignableInstanceManager();
  prevAssignableInstanceManager.buildAssignableInstances(clusterConfig, taskDataCache,
      liveInstanceMap, instanceConfigMap);
  Map<String, AssignableInstance> prevAssignableInstanceMap =
      new HashMap<>(prevAssignableInstanceManager.getAssignableInstanceMap());
  Map<String, TaskAssignResult> prevTaskAssignResultMap =
      new HashMap<>(prevAssignableInstanceManager.getTaskAssignResultMap());

  // Generate a new AssignableInstanceManager
  taskDataCache.refresh(accessor, resourceConfigMap);
  AssignableInstanceManager newAssignableInstanceManager = new AssignableInstanceManager();
  newAssignableInstanceManager.buildAssignableInstances(clusterConfig, taskDataCache,
      liveInstanceMap, instanceConfigMap);
  Map<String, AssignableInstance> newAssignableInstanceMap =
      new HashMap<>(newAssignableInstanceManager.getAssignableInstanceMap());
  Map<String, TaskAssignResult> newTaskAssignResultMap =
      new HashMap<>(newAssignableInstanceManager.getTaskAssignResultMap());

  // Compare prev and new - they should match up exactly
  Assert.assertEquals(prevAssignableInstanceMap.size(), newAssignableInstanceMap.size());
  Assert.assertEquals(prevTaskAssignResultMap.size(), newTaskAssignResultMap.size());
  for (Map.Entry<String, AssignableInstance> assignableInstanceEntry : newAssignableInstanceMap
      .entrySet()) {
    String instance = assignableInstanceEntry.getKey();
    Assert.assertEquals(prevAssignableInstanceMap.get(instance).getCurrentAssignments(),
        assignableInstanceEntry.getValue().getCurrentAssignments());
    Assert.assertEquals(prevAssignableInstanceMap.get(instance).getTotalCapacity(),
        assignableInstanceEntry.getValue().getTotalCapacity());
    Assert.assertEquals(prevAssignableInstanceMap.get(instance).getUsedCapacity(),
        assignableInstanceEntry.getValue().getUsedCapacity());
  }
  for (Map.Entry<String, TaskAssignResult> taskAssignResultEntry : newTaskAssignResultMap
      .entrySet()) {
    String taskID = taskAssignResultEntry.getKey();
    Assert.assertEquals(prevTaskAssignResultMap.get(taskID).toString(),
        taskAssignResultEntry.getValue().toString());
  }

  // Shut down RoutingTableProvider so periodic update gets shut down
  routingTableProvider.shutdown();
}