org.apache.helix.model.InstanceConfig Java Examples

@Before
public void setup() {
  MockitoAnnotations.initMocks(this);
  this.testRoutingTableProvider = new TestRoutingTableProvider(null);
  this.hdfsManager = new HdfsManager(new PropertiesConfiguration(),
      mockZkManager,
      CLUSTER_NAME,
      mockHelixAdmin,
      mockDfsClient,
      testRoutingTableProvider);
  hostConfig1 = new InstanceConfig("host1");
  hostConfig1.setHostName("host1");
  hostConfig2 = new InstanceConfig("host2");
  hostConfig2.setHostName("host2");
  hostConfig3 = new InstanceConfig("host3");
  hostConfig3.setHostName("host3");
  hostConfig4 = new InstanceConfig("host4");
  hostConfig4.setHostName("host4");
}
 

public ClusterModel getMultiNodeClusterModel() throws IOException {
  initialize();
  ResourceControllerDataProvider testCache = setupClusterDataCache();
  InstanceConfig testInstanceConfig1 = createMockInstanceConfig(TEST_INSTANCE_ID_1);
  InstanceConfig testInstanceConfig2 = createMockInstanceConfig(TEST_INSTANCE_ID_2);
  Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();
  instanceConfigMap.put(TEST_INSTANCE_ID_1, testInstanceConfig1);
  instanceConfigMap.put(TEST_INSTANCE_ID_2, testInstanceConfig2);
  when(testCache.getInstanceConfigMap()).thenReturn(instanceConfigMap);
  Set<AssignableReplica> assignableReplicas = generateReplicas(testCache);
  Set<AssignableNode> assignableNodes = generateNodes(testCache);

  ClusterContext context =
      new ClusterContext(assignableReplicas, assignableNodes, Collections.emptyMap(), Collections.emptyMap());
  return new ClusterModel(context, assignableReplicas, assignableNodes);
}
 

@PUT
public Response addInstance(@PathParam("clusterId") String clusterId,
    @PathParam("instanceName") String instanceName, String content) {
  HelixAdmin admin = getHelixAdmin();
  ZNRecord record;
  try {
    record = toZNRecord(content);
  } catch (IOException e) {
    LOG.error("Failed to deserialize user's input " + content + ", Exception: " + e);
    return badRequest("Input is not a vaild ZNRecord!");
  }

  try {
    admin.addInstance(clusterId, new InstanceConfig(record));
  } catch (Exception ex) {
    LOG.error("Error in adding an instance: " + instanceName, ex);
    return serverError(ex);
  }

  return OK();
}
 

@Test(dataProvider = "isEnabledTestSuite")
public void TestIsInstanceEnabled(boolean instanceConfigEnabled, boolean clusterConfigEnabled,
    boolean expected) {
  Mock mock = new Mock();
  InstanceConfig instanceConfig = new InstanceConfig(TEST_INSTANCE);
  instanceConfig.setInstanceEnabled(instanceConfigEnabled);
  doReturn(instanceConfig).when(mock.dataAccessor)
      .getProperty(BUILDER.instanceConfig(TEST_INSTANCE));
  ClusterConfig clusterConfig = new ClusterConfig(TEST_CLUSTER);
  if (!clusterConfigEnabled) {
    clusterConfig.setDisabledInstances(ImmutableMap.of(TEST_INSTANCE, "12345"));
  }
  doReturn(clusterConfig).when(mock.dataAccessor)
      .getProperty(BUILDER.clusterConfig());

  boolean isEnabled = InstanceValidationUtil.isEnabled(mock.dataAccessor, TEST_INSTANCE);

  Assert.assertEquals(isEnabled, expected);
}
 

@Test
public void testLegacyEnableDisablePartition() {
  String instanceName = "TestInstanceLegacy";
  String testResourcePrefix = "TestResourceLegacy";
  ZNRecord record = new ZNRecord(instanceName);
  List<String> disabledPartitions = new ArrayList<>(Arrays.asList("1", "2", "3"));
  record.setListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name(),
      disabledPartitions);
  InstanceConfig instanceConfig = new InstanceConfig(record);
  instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", false);
  Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 3);
  Assert.assertEquals(instanceConfig.getRecord()
          .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
      3);
  instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", true);
  Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 2);
  Assert.assertEquals(instanceConfig.getRecord()
          .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
      2);
}
 

@GET
@Path("/instances/{instanceName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get instance information", produces = MediaType.APPLICATION_JSON)
@ApiResponses(value = {@ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Instance not found"), @ApiResponse(code = 500, message = "Internal error")})
public String getInstance(
    @ApiParam(value = "Instance name", required = true, example = "Server_a.b.com_20000 | Broker_my.broker.com_30000") @PathParam("instanceName") String instanceName) {
  InstanceConfig instanceConfig = pinotHelixResourceManager.getHelixInstanceConfig(instanceName);
  if (instanceConfig == null) {
    throw new ControllerApplicationException(LOGGER, "Instance " + instanceName + " not found",
        Response.Status.NOT_FOUND);
  }
  ObjectNode response = JsonUtils.newObjectNode();
  response.put("instanceName", instanceConfig.getInstanceName());
  response.put("hostName", instanceConfig.getHostName());
  response.put("enabled", instanceConfig.getInstanceEnabled());
  response.put("port", instanceConfig.getPort());
  response.set("tags", JsonUtils.objectToJsonNode(instanceConfig.getTags()));
  response.set("pools", JsonUtils.objectToJsonNode(instanceConfig.getRecord().getMapField(InstanceUtils.POOL_KEY)));
  return response.toString();
}
 

@Test
public void testInitializationWithOnlyCapacity() {
  // Initialize AssignableInstance with only resource capacity provided
  AssignableInstance ai = new AssignableInstance(createClusterConfig(null, null, false),
      new InstanceConfig(testInstanceName),
      createLiveInstance(testResourceTypes, testResourceCapacity));
  Assert.assertEquals(ai.getTotalCapacity().size(), testResourceTypes.length);
  Assert.assertEquals(ai.getUsedCapacity().size(), testResourceTypes.length);
  for (int i = 0; i < testResourceTypes.length; i++) {
    Assert.assertEquals(ai.getTotalCapacity().get(testResourceTypes[i]).size(), 1);
    Assert.assertEquals(ai.getUsedCapacity().get(testResourceTypes[i]).size(), 1);
    Assert.assertEquals(ai.getTotalCapacity().get(testResourceTypes[i])
        .get(AssignableInstance.DEFAULT_QUOTA_TYPE), Integer.valueOf(testResourceCapacity[i]));
    Assert.assertEquals(
        ai.getUsedCapacity().get(testResourceTypes[i]).get(AssignableInstance.DEFAULT_QUOTA_TYPE),
        Integer.valueOf(0));
  }
}
 

@Test
public void testGetClusterTopology_whenMultiZones() {
  InstanceConfig instanceConfig1 = new InstanceConfig("instance0");
  instanceConfig1.setDomain("helixZoneId=zone0");
  InstanceConfig instanceConfig2 = new InstanceConfig("instance1");
  instanceConfig2.setDomain("helixZoneId=zone1");
  List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1, instanceConfig2);

  Mock mock = new Mock();
  when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER));
  when(mock.dataAccessor.getChildValues(any(PropertyKey.class), anyBoolean())).thenReturn(instanceConfigs);

  ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER);

  Assert.assertEquals(clusterTopology.getZones().size(), 2);
  Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER);
}
 

/**
 * returns all instances for resources contains any given tags in {resource group} that are in a
 * specific {state}
 * @param resourceGroupName
 * @param state
 * @return empty list if there is no instance in a given state
 */
public Set<InstanceConfig> getInstancesForResourceGroup(String resourceGroupName, String state,
    List<String> resourceTags) {
  Set<InstanceConfig> instanceSet = null;
  ResourceGroupInfo resourceGroupInfo = getResourceGroup(resourceGroupName);
  if (resourceGroupInfo != null) {
    instanceSet = new HashSet<>();
    for (String tag : resourceTags) {
      Set<InstanceConfig> instances = resourceGroupInfo.getInstances(state, tag);
      if (instances != null) {
        instanceSet.addAll(resourceGroupInfo.getInstances(state, tag));
      }
    }
  }
  if (instanceSet == null) {
    return Collections.emptySet();
  }
  return instanceSet;
}
 

private boolean compare(IdealState idealState, RoutingTableProvider routingTableEV,
    RoutingTableProvider routingTableCurrentStates) {
  String db = idealState.getResourceName();
  Set<String> partitions = idealState.getPartitionSet();
  for (String partition : partitions) {
    List<InstanceConfig> masterInsEv =
        routingTableEV.getInstancesForResource(db, partition, "MASTER");
    List<InstanceConfig> masterInsCs =
        routingTableCurrentStates.getInstancesForResource(db, partition, "MASTER");
    if (masterInsEv.size() != 1 || masterInsCs.size() != 1 || !masterInsCs.equals(masterInsEv)) {
      return false;
    }
    List<InstanceConfig> slaveInsEv =
        routingTableEV.getInstancesForResource(db, partition, "SLAVE");
    List<InstanceConfig> slaveInsCs =
        routingTableCurrentStates.getInstancesForResource(db, partition, "SLAVE");
    if (slaveInsEv.size() != 2 || slaveInsCs.size() != 2 || !new HashSet(slaveInsCs)
        .equals(new HashSet(slaveInsEv))) {
      return false;
    }
  }
  return true;
}
 

/**
 * Validates and returns instance capacities. The validation logic ensures that all required capacity keys (in ClusterConfig) are present in InstanceConfig.
 * @param clusterConfig
 * @param instanceConfig
 * @return
 */
public static Map<String, Integer> validateAndGetInstanceCapacity(ClusterConfig clusterConfig,
    InstanceConfig instanceConfig) {
  // Fetch the capacity of instance from 2 possible sources according to the following priority.
  // 1. The instance capacity that is configured in the instance config.
  // 2. If the default instance capacity that is configured in the cluster config contains more capacity keys, fill the capacity map with those additional values.
  Map<String, Integer> instanceCapacity =
      new HashMap<>(clusterConfig.getDefaultInstanceCapacityMap());
  instanceCapacity.putAll(instanceConfig.getInstanceCapacityMap());

  List<String> requiredCapacityKeys = clusterConfig.getInstanceCapacityKeys();
  // All the required keys must exist in the instance config.
  if (!instanceCapacity.keySet().containsAll(requiredCapacityKeys)) {
    throw new HelixException(String.format(
        "The required capacity keys: %s are not fully configured in the instance: %s, capacity map: %s.",
        requiredCapacityKeys.toString(), instanceConfig.getInstanceName(),
        instanceCapacity.toString()));
  }
  return instanceCapacity;
}
 

/**
 * returns the instances for {resource,partition} pair that are in a specific
 * {state}
 * @param resourceName
 * @param partitionName
 * @param state
 * @return empty list if there is no instance in a given state
 */
public List<InstanceConfig> getInstancesForResource(String resourceName, String partitionName,
    String state) {
  List<InstanceConfig> instanceList = null;
  ResourceInfo resourceInfo = get(resourceName);
  if (resourceInfo != null) {
    PartitionInfo keyInfo = resourceInfo.get(partitionName);
    if (keyInfo != null) {
      instanceList = keyInfo.get(state);
    }
  }
  if (instanceList == null) {
    instanceList = Collections.emptyList();
  }
  return instanceList;
}
 

@Override
public Map<Integer, List<InstanceConfig>> applyConstraint(
    Map<Integer, List<InstanceConfig>> poolToInstanceConfigsMap) {
  int tableNameHash = Math.abs(_tableNameWithType.hashCode());
  LOGGER.info("Rotating instances for table: {} with hash: {}", _tableNameWithType, tableNameHash);

  for (Map.Entry<Integer, List<InstanceConfig>> entry : poolToInstanceConfigsMap.entrySet()) {
    List<InstanceConfig> instanceConfigs = entry.getValue();
    int numInstanceConfigs = instanceConfigs.size();
    Preconditions.checkState(numInstanceConfigs > 0, "No instance left in pool: %s", entry.getKey());

    // This should match the default InstancePartitions generated from InstancePartitionsUtils
    Collections.rotate(instanceConfigs, -(tableNameHash % numInstanceConfigs));
  }

  return poolToInstanceConfigsMap;
}
 

private static void addNode() throws Exception {

    NUM_NODES = NUM_NODES + 1;
    int port = 12000 + NUM_NODES - 1;
    InstanceConfig instanceConfig = new InstanceConfig("localhost_" + port);
    instanceConfig.setHostName("localhost");
    instanceConfig.setPort("" + port);
    instanceConfig.setInstanceEnabled(true);
    echo("ADDING NEW NODE :" + instanceConfig.getInstanceName()
        + ". Partitions will move from old nodes to the new node.");
    admin.addInstance(CLUSTER_NAME, instanceConfig);
    INSTANCE_CONFIG_LIST.add(instanceConfig);
    MyProcess process = new MyProcess(instanceConfig.getInstanceName());
    PROCESS_LIST.add(process);
    admin.rebalance(CLUSTER_NAME, RESOURCE_NAME, 3);
    process.start();
  }
 

@Test(dataProvider = "IdealStateInput")
public void testIdealStateAssignment(String clusterName, List<String> instances,
    List<String> partitions, String numReplicas, String stateModeDef, String strategyName,
    Map<String, Map<String, String>> expectedMapping, List<String> disabledInstances)
    throws IllegalAccessException, InstantiationException, ClassNotFoundException {
  ClusterConfig clusterConfig = new ClusterConfig(clusterName);
  List<InstanceConfig> instanceConfigs = new ArrayList<>();
  for (String instance : instances) {
    instanceConfigs.add(new InstanceConfig(instance));
    if (disabledInstances.contains(instance)) {
      instanceConfigs.get(instanceConfigs.size() - 1).setInstanceEnabled(false);
    }
  }

  IdealState idealState = new IdealState("TestResource");
  idealState.setStateModelDefRef(stateModeDef);
  idealState.setNumPartitions(partitions.size());
  idealState.setReplicas(numReplicas);

  Map<String, Map<String, String>> idealStateMapping = HelixUtil
      .getIdealAssignmentForFullAuto(clusterConfig, instanceConfigs, instances, idealState,
          partitions, strategyName);
  Assert.assertEquals(idealStateMapping, expectedMapping);
}
 

@Override
public List<String> getInstancesInClusterWithTag(String clusterName, String tag) {
  String memberInstancesPath = PropertyPathBuilder.instance(clusterName);
  List<String> instances = _zkClient.getChildren(memberInstancesPath);
  List<String> result = new ArrayList<String>();

  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();

  for (String instanceName : instances) {
    InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceName));
    if (config == null) {
      throw new IllegalStateException(String
          .format("Instance %s does not have a config, cluster might be in bad state",
              instanceName));
    }
    if (config.containsTag(tag)) {
      result.add(instanceName);
    }
  }
  return result;
}
 

@Override
public void setInstanceZoneId(String clusterName, String instanceName, String zoneId) {
  logger.info("Set instance zoneId {} for instance {} in cluster {}.", zoneId, instanceName,
      clusterName);
  if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
    throw new HelixException("cluster " + clusterName + " is not setup yet");
  }

  if (!ZKUtil.isInstanceSetup(_zkClient, clusterName, instanceName, InstanceType.PARTICIPANT)) {
    throw new HelixException(
        "cluster " + clusterName + " instance " + instanceName + " is not setup yet");
  }
  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();

  InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceName));
  config.setZoneId(zoneId);
  accessor.setProperty(keyBuilder.instanceConfig(instanceName), config);
}
 

@Override
public Map<String, Boolean> validateInstancesForWagedRebalance(String clusterName,
    List<String> instanceNames) {
  // Null checks
  if (clusterName == null || clusterName.isEmpty()) {
    throw new HelixException("Cluster name is invalid!");
  }
  if (instanceNames == null || instanceNames.isEmpty()) {
    throw new HelixException("Instance name list is invalid!");
  }

  Map<String, Boolean> result = new HashMap<>();
  ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
  for (String instanceName : instanceNames) {
    InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(clusterName, instanceName);
    if (instanceConfig == null || !instanceConfig.isValid()) {
      result.put(instanceName, false);
      continue;
    }
    WagedValidationUtil.validateAndGetInstanceCapacity(clusterConfig, instanceConfig);
    result.put(instanceName, true);
  }

  return result;
}
 

@Override
protected synchronized void refreshExternalView(Collection<ExternalView> externalViews,
    Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
    String referenceKey) {
  super.refreshExternalView(externalViews, instanceConfigs, liveInstances, referenceKey);
  _refreshCount++;
  if (DEBUG) {
    print();
  }
}
 

@Override
@PreFetch(enabled = false)
public void onInstanceConfigChange(List<InstanceConfig> configs,
    NotificationContext changeContext) {
  _routerUpdater.queueEvent(changeContext, ClusterEventType.InstanceConfigChange,
      HelixConstants.ChangeType.INSTANCE_CONFIG);
}
 

protected void refreshCurrentState(Map<String, Map<String, Map<String, CurrentState>>> currentStateMap,
    Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
    String referenceKey) {
  long startTime = System.currentTimeMillis();
  RoutingTable newRoutingTable =
      new RoutingTable(currentStateMap, instanceConfigs, liveInstances);
  resetRoutingTableAndNotify(startTime, newRoutingTable, referenceKey);
}
 

@Override
@PreFetch(enabled = false)
public void onInstanceConfigChange(List<InstanceConfig> instanceConfigs,
    NotificationContext changeContext) {
  logger.info(
      "START: GenericClusterController.onInstanceConfigChange() for cluster " + _clusterName);
  notifyCaches(changeContext, ChangeType.INSTANCE_CONFIG);
  pushToEventQueues(ClusterEventType.InstanceConfigChange, changeContext,
      Collections.<String, Object>emptyMap());
  logger.info(
      "END: GenericClusterController.onInstanceConfigChange() for cluster " + _clusterName);
}
 

private static InstanceConfig getInstanceCfg(ParticipantConfig participant) {
  if (participant == null || participant.name == null || participant.host == null
      || participant.port == null) {
    throw new HelixException("Participant must have a specified name, host, and port!");
  }
  InstanceConfig instanceCfg = new InstanceConfig(participant.name);
  instanceCfg.setHostName(participant.host);
  instanceCfg.setPort(participant.port.toString());
  return instanceCfg;
}
 

protected void refreshExternalView(Collection<ExternalView> externalViews,
    Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
    String referenceKey) {
  long startTime = System.currentTimeMillis();
  PropertyType propertyType = _routingTableRefMap.get(referenceKey).get().getPropertyType();
  RoutingTable newRoutingTable =
      new RoutingTable(externalViews, instanceConfigs, liveInstances, propertyType);
  resetRoutingTableAndNotify(startTime, newRoutingTable, referenceKey);
}
 

protected CustomizedViewRoutingTable(Collection<CustomizedView> customizedViews,
    Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances,
    PropertyType propertytype, String customizedStateType) {
  super(Collections.<ExternalView> emptyList(), instanceConfigs, liveInstances,
      PropertyType.CUSTOMIZEDVIEW);
  _customizedStateType = customizedStateType;
  _customizedViews = new HashSet<>(customizedViews);
  refresh(_customizedViews);
}
 

private void setParticipantsCapacity(int perNodeTaskLimitation) {
  for (int i = 0; i < _numNodes; i++) {
    InstanceConfig instanceConfig = _gSetupTool.getClusterManagementTool()
        .getInstanceConfig(CLUSTER_NAME, PARTICIPANT_PREFIX + "_" + (_startPort + i));
    instanceConfig.setMaxConcurrentTask(perNodeTaskLimitation);
    _gSetupTool.getClusterManagementTool().setInstanceConfig(CLUSTER_NAME,
        PARTICIPANT_PREFIX + "_" + (_startPort + i), instanceConfig);
  }
}
 

@Override
public int compare(InstanceConfig config1, InstanceConfig config2) {
  if (config1 == config2) {
    return 0;
  }
  if (config1 == null) {
    return -1;
  }
  if (config2 == null) {
    return 1;
  }
  // HELIX-936: a NPE on the hostname; compare IDs instead. IDs for InstanceConfigs are
  // concatenation of instance name, host, and port.
  return config1.getId().compareTo(config2.getId());
}
 

protected void setupInstances(String clusterName, int[] instances) {
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  for (int i = 0; i < instances.length; i++) {
    String instance = "localhost_" + instances[i];
    InstanceConfig instanceConfig = new InstanceConfig(instance);
    instanceConfig.setHostName("localhost");
    instanceConfig.setPort("" + instances[i]);
    instanceConfig.setInstanceEnabled(true);
    admin.addInstance(clusterName, instanceConfig);
  }
}
 

public Set<InstanceConfig> getInstances(String state, String resourceTag) {
  ResourceInfo resourceInfo = tagToResourceMap.get(resourceTag);
  if (resourceInfo != null) {
    return resourceInfo.getInstances(state);
  }

  return null;
}
 

public void addEntry(String state, InstanceConfig config) {
  if (!stateInfoMap.containsKey(state)) {
    stateInfoMap.put(state, new ArrayList<InstanceConfig>());
  }
  List<InstanceConfig> list = stateInfoMap.get(state);
  list.add(config);
}