Java Code Examples for org.apache.helix.model.InstanceConfig#setInstanceEnabled()

public synchronized void start() throws Exception{
  LOGGER.info("Trying to start ManagerControllerHelix!");
  _helixZkManager = HelixManagerFactory.getZKHelixManager(_helixClusterName,
      _instanceId,
      InstanceType.PARTICIPANT,
      _helixZkURL);
  _helixZkManager.getStateMachineEngine().registerStateModelFactory("OnlineOffline",
      new ControllerStateModelFactory(this));
  try {
    _helixZkManager.connect();
    InstanceConfig instanceConfig = new InstanceConfig(_instanceId);
    instanceConfig.setHostName(_hostname);
    instanceConfig.setPort(_port);
    instanceConfig.setInstanceEnabled(true);
    _helixZkManager.getConfigAccessor().setInstanceConfig(_helixClusterName, _instanceId,
        instanceConfig);
  } catch (Exception e) {
    LOGGER.error("Failed to start ManagerControllerHelix " + _helixClusterName, e);
    throw e;
  }
}
 

public void start() throws Exception {
  LOGGER.info("Starting MangerWorkerHelixManager");
  try {
    mangerWorkerHelixManager.connect();
  } catch (Exception e) {
    LOGGER.error("Connecting to mangerWorkerHelixManager failed", e);
    throw e;
  }
  mangerWorkerHelixManager.getStateMachineEngine().registerStateModelFactory("OnlineOffline",
      new OnlineOfflineStateFactory(this));
  InstanceConfig instanceConfig = new InstanceConfig(instanceId);
  instanceConfig.setHostName(workerConf.getHostname());
  instanceConfig.setInstanceEnabled(true);
  mangerWorkerHelixManager.getConfigAccessor()
      .setInstanceConfig(clusterName, instanceId,
          instanceConfig);
  LOGGER.info("Register MangerWorkerHelixManager finished");
}
 

private static void addNode() throws Exception {

    NUM_NODES = NUM_NODES + 1;
    int port = 12000 + NUM_NODES - 1;
    InstanceConfig instanceConfig = new InstanceConfig("localhost_" + port);
    instanceConfig.setHostName("localhost");
    instanceConfig.setPort("" + port);
    instanceConfig.setInstanceEnabled(true);
    echo("ADDING NEW NODE :" + instanceConfig.getInstanceName()
        + ". Partitions will move from old nodes to the new node.");
    admin.addInstance(CLUSTER_NAME, instanceConfig);
    INSTANCE_CONFIG_LIST.add(instanceConfig);
    MyProcess process = new MyProcess(instanceConfig.getInstanceName());
    PROCESS_LIST.add(process);
    admin.rebalance(CLUSTER_NAME, RESOURCE_NAME, 3);
    process.start();
  }
 

/**
 * Init the rebalance util with cluster and instances information.
 *
 * Note that it is not required to put any configuration items in these configs.
 * However, in order to do topology aware rebalance, users need to set topology information such as Domain, fault zone, and TopologyAwareEnabled.
 *
 * The other config items will not be read or processed by the util.
 *
 * @param clusterConfig
 * @param instanceConfigs InstanceConfigs for all assignment candidates.
 *                        Note that all instances will be treated as enabled and alive during the calculation.
 */
public WeightAwareRebalanceUtil(ClusterConfig clusterConfig,
    List<InstanceConfig> instanceConfigs) {
  for (InstanceConfig instanceConfig : instanceConfigs) {
    // ensure the instance is enabled
    instanceConfig.setInstanceEnabled(true);
    _instanceConfigMap.put(instanceConfig.getInstanceName(), instanceConfig);
  }
  // ensure no instance is disabled
  clusterConfig.setDisabledInstances(Collections.<String, String>emptyMap());
  _clusterConfig = clusterConfig;

  _dataCache = new ResourceControllerDataProvider();
  _dataCache.setInstanceConfigMap(_instanceConfigMap);
  _dataCache.setClusterConfig(_clusterConfig);
  List<LiveInstance> liveInstanceList = new ArrayList<>();
  for (String instance : _instanceConfigMap.keySet()) {
    LiveInstance liveInstance = new LiveInstance(instance);
    liveInstanceList.add(liveInstance);
  }
  _dataCache.setLiveInstances(liveInstanceList);
}
 

private static void addInstanceConfig(String instanceName) {
  // add node to cluster if not already added
  ZKHelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);

  InstanceConfig instanceConfig = null;
  try {
    instanceConfig = admin.getInstanceConfig(_clusterName, instanceName);
  } catch (Exception ignored) {
  }
  if (instanceConfig == null) {
    InstanceConfig config = new InstanceConfig(instanceName);
    config.setHostName("localhost");
    config.setInstanceEnabled(true);
    echo("Adding InstanceConfig:" + config);
    admin.addInstance(_clusterName, config);
  }
}
 

@Test(dataProvider = "isEnabledTestSuite")
public void TestIsInstanceEnabled(boolean instanceConfigEnabled, boolean clusterConfigEnabled,
    boolean expected) {
  Mock mock = new Mock();
  InstanceConfig instanceConfig = new InstanceConfig(TEST_INSTANCE);
  instanceConfig.setInstanceEnabled(instanceConfigEnabled);
  doReturn(instanceConfig).when(mock.dataAccessor)
      .getProperty(BUILDER.instanceConfig(TEST_INSTANCE));
  ClusterConfig clusterConfig = new ClusterConfig(TEST_CLUSTER);
  if (!clusterConfigEnabled) {
    clusterConfig.setDisabledInstances(ImmutableMap.of(TEST_INSTANCE, "12345"));
  }
  doReturn(clusterConfig).when(mock.dataAccessor)
      .getProperty(BUILDER.clusterConfig());

  boolean isEnabled = InstanceValidationUtil.isEnabled(mock.dataAccessor, TEST_INSTANCE);

  Assert.assertEquals(isEnabled, expected);
}
 

/**
 * Compose the config for an instance
 * @param instanceName
 * @return InstanceConfig
 */
public static InstanceConfig composeInstanceConfig(String instanceName) {
  InstanceConfig instanceConfig = new InstanceConfig(instanceName);
  String hostName = instanceName;
  String port = "";
  int lastPos = instanceName.lastIndexOf("_");
  if (lastPos > 0) {
    hostName = instanceName.substring(0, lastPos);
    port = instanceName.substring(lastPos + 1);
  }
  instanceConfig.setHostName(hostName);
  instanceConfig.setPort(port);
  instanceConfig.setInstanceEnabled(true);
  return instanceConfig;
}
 

protected InstanceConfig createMockInstanceConfig(String instanceId) {
  InstanceConfig testInstanceConfig = new InstanceConfig(instanceId);
  testInstanceConfig.setInstanceCapacityMap(_capacityDataMap);
  testInstanceConfig.addTag(_testInstanceTags.get(0));
  testInstanceConfig.setInstanceEnabled(true);
  testInstanceConfig.setZoneId(_testFaultZoneId);
  return testInstanceConfig;
}
 

protected void setupInstances(String clusterName, int[] instances) {
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  for (int i = 0; i < instances.length; i++) {
    String instance = "localhost_" + instances[i];
    InstanceConfig instanceConfig = new InstanceConfig(instance);
    instanceConfig.setHostName("localhost");
    instanceConfig.setPort("" + instances[i]);
    instanceConfig.setInstanceEnabled(true);
    admin.addInstance(clusterName, instanceConfig);
  }
}
 

@Override
public void run() {
  ZkClient zkclient = null;
  try {
    // add node to cluster if not already added
    zkclient =
        new ZkClient(_zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT,
            ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);

    List<String> nodes = admin.getInstancesInCluster(_clusterName);
    if (!nodes.contains(_instanceName)) {
      InstanceConfig config = new InstanceConfig(_instanceName);
      config.setHostName("localhost");
      config.setInstanceEnabled(true);
      admin.addInstance(_clusterName, config);
    }

    Runtime.getRuntime().addShutdownHook(new Thread() {
      @Override
      public void run() {
        System.out.println("Shutting down " + _instanceName);
        disconnect();
      }
    });

    connect();
  } finally {
    if (zkclient != null) {
      zkclient.close();
    }
  }
}
 

@Test(dependsOnMethods = "testInstancesStoppable_zoneBased")
public void testInstancesStoppable_disableOneInstance() throws IOException {
  // Disable one selected instance0, it should failed to check
  String instance = "instance0";
  InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(STOPPABLE_CLUSTER, instance);
  instanceConfig.setInstanceEnabled(false);
  instanceConfig.setInstanceEnabledForPartition("FakeResource", "FakePartition", false);
  _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER, instance, instanceConfig);

  // It takes time to reflect the changes.
  BestPossibleExternalViewVerifier verifier =
      new BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER).setZkAddr(ZK_ADDR).build();
  Assert.assertTrue(verifier.verifyByPolling());

  Entity entity = Entity.entity("", MediaType.APPLICATION_JSON_TYPE);
  Response response = new JerseyUriRequestBuilder("clusters/{}/instances/{}/stoppable")
      .format(STOPPABLE_CLUSTER, instance).post(this, entity);
  JsonNode jsonResult = OBJECT_MAPPER.readTree(response.readEntity(String.class));
  Assert.assertFalse(jsonResult.get("stoppable").asBoolean());
  Assert.assertEquals(getStringSet(jsonResult, "failedChecks"),
          ImmutableSet.of("HELIX:HAS_DISABLED_PARTITION","HELIX:INSTANCE_NOT_ENABLED","HELIX:INSTANCE_NOT_STABLE","HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));

  // Reenable instance0, it should passed the check
  instanceConfig.setInstanceEnabled(true);
  instanceConfig.setInstanceEnabledForPartition("FakeResource", "FakePartition", true);
  _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER, instance, instanceConfig);
  Assert.assertTrue(verifier.verifyByPolling());

  entity = Entity.entity("", MediaType.APPLICATION_JSON_TYPE);
  response = new JerseyUriRequestBuilder("clusters/{}/instances/{}/stoppable")
      .format(STOPPABLE_CLUSTER, instance).post(this, entity);
  jsonResult = OBJECT_MAPPER.readTree(response.readEntity(String.class));

  Assert.assertFalse(jsonResult.get("stoppable").asBoolean());
  Assert.assertEquals(getStringSet(jsonResult, "failedChecks"), ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
  System.out.println("End test :" + TestHelper.getTestMethodName());
}
 

@Test
public void testRebuildBrokerResourceWhenBrokerAdded()
    throws Exception {
  // Check that the first table we added doesn't need to be rebuilt(case where ideal state brokers and brokers in broker resource are the same.
  String partitionName = _offlineTableConfig.getTableName();
  HelixAdmin helixAdmin = _helixManager.getClusterManagmentTool();

  IdealState idealState = HelixHelper.getBrokerIdealStates(helixAdmin, getHelixClusterName());
  // Ensure that the broker resource is not rebuilt.
  Assert.assertTrue(idealState.getInstanceSet(partitionName)
      .equals(_helixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME)));
  _helixResourceManager.rebuildBrokerResourceFromHelixTags(partitionName);

  // Add another table that needs to be rebuilt
  TableConfig offlineTableConfigTwo = new TableConfigBuilder(TableType.OFFLINE).setTableName(TEST_TABLE_TWO).build();
  _helixResourceManager.addTable(offlineTableConfigTwo);
  String partitionNameTwo = offlineTableConfigTwo.getTableName();

  // Add a new broker manually such that the ideal state is not updated and ensure that rebuild broker resource is called
  final String brokerId = "Broker_localhost_2";
  InstanceConfig instanceConfig = new InstanceConfig(brokerId);
  instanceConfig.setInstanceEnabled(true);
  instanceConfig.setHostName("Broker_localhost");
  instanceConfig.setPort("2");
  helixAdmin.addInstance(getHelixClusterName(), instanceConfig);
  helixAdmin.addInstanceTag(getHelixClusterName(), instanceConfig.getInstanceName(),
      TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME));
  idealState = HelixHelper.getBrokerIdealStates(helixAdmin, getHelixClusterName());
  // Assert that the two don't equal before the call to rebuild the broker resource.
  Assert.assertTrue(!idealState.getInstanceSet(partitionNameTwo)
      .equals(_helixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME)));
  _helixResourceManager.rebuildBrokerResourceFromHelixTags(partitionNameTwo);
  idealState = HelixHelper.getBrokerIdealStates(helixAdmin, getHelixClusterName());
  // Assert that the two do equal after being rebuilt.
  Assert.assertTrue(idealState.getInstanceSet(partitionNameTwo)
      .equals(_helixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME)));
}
 

private void setInstanceEnable(String instanceName, boolean enabled,
    ConfigAccessor configAccessor) {
  InstanceConfig instanceConfig = configAccessor.getInstanceConfig(CLUSTER_NAME, instanceName);
  instanceConfig.setInstanceEnabled(enabled);
  configAccessor.setInstanceConfig(CLUSTER_NAME, instanceName, instanceConfig);
}
 

/**
 * Modify IdealState mapping fields for a FULL_AUTO resource and see if detector detects.
 */
@Test(dependsOnMethods = "testNoChange")
public void testIgnoreNonTopologyChanges() {
  // Modify cluster config and IdealState to ensure the mapping field of the IdealState will be
  // considered as the fields that are modified by Helix logic.
  ClusterConfig clusterConfig = _dataAccessor.getProperty(_keyBuilder.clusterConfig());
  clusterConfig.setPersistBestPossibleAssignment(true);
  _dataAccessor.updateProperty(_keyBuilder.clusterConfig(), clusterConfig);

  // Create an new IS
  String resourceName = "Resource" + TestHelper.getTestMethodName();
  _gSetupTool.getClusterManagementTool()
      .addResource(CLUSTER_NAME, resourceName, NUM_PARTITIONS, STATE_MODEL);
  IdealState idealState = _dataAccessor.getProperty(_keyBuilder.idealStates(resourceName));
  idealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
  idealState.getRecord().getMapFields().put("Partition1", new HashMap<>());
  _dataAccessor.updateProperty(_keyBuilder.idealStates(resourceName), idealState);
  Arrays.stream(ChangeType.values()).forEach(type -> {
    _dataProvider.notifyDataChange(type);
  });
  _dataProvider.refresh(_dataAccessor);

  // Test with ignore option to be true
  ResourceChangeDetector changeDetector = new ResourceChangeDetector(true);
  changeDetector.updateSnapshots(_dataProvider);

  // 1. Modify ideal state map fields of a FULL_AUTO resource
  idealState.getRecord().getMapFields().put("Partition1", Collections.singletonMap("foo", "bar"));
  _dataAccessor.updateProperty(_keyBuilder.idealStates(resourceName), idealState);
  _dataProvider.notifyDataChange(ChangeType.IDEAL_STATE);
  _dataProvider.refresh(_dataAccessor);
  changeDetector.updateSnapshots(_dataProvider);
  Assert.assertEquals(changeDetector.getChangeTypes(),
      Collections.singleton(ChangeType.IDEAL_STATE));
  Assert.assertEquals(
      changeDetector.getAdditionsByType(ChangeType.IDEAL_STATE).size() + changeDetector
          .getChangesByType(ChangeType.IDEAL_STATE).size() + changeDetector
          .getRemovalsByType(ChangeType.IDEAL_STATE).size(), 0);

  // 2. Modify an instance "enabled" state
  String instanceName = _participants[0].getInstanceName();
  InstanceConfig instanceConfig =
      _dataAccessor.getProperty(_keyBuilder.instanceConfig(instanceName));
  Assert.assertTrue(instanceConfig.getInstanceEnabled());
  try {
    instanceConfig.setInstanceEnabled(false);
    _dataAccessor.updateProperty(_keyBuilder.instanceConfig(instanceName), instanceConfig);
    _dataProvider.notifyDataChange(ChangeType.INSTANCE_CONFIG);
    _dataProvider.refresh(_dataAccessor);
    changeDetector.updateSnapshots(_dataProvider);
    Assert.assertEquals(changeDetector.getChangeTypes(),
        Collections.singleton(ChangeType.INSTANCE_CONFIG));
    Assert.assertEquals(
        changeDetector.getAdditionsByType(ChangeType.INSTANCE_CONFIG).size() + changeDetector
            .getChangesByType(ChangeType.INSTANCE_CONFIG).size() + changeDetector
            .getRemovalsByType(ChangeType.INSTANCE_CONFIG).size(), 0);
  } finally {
    instanceConfig.setInstanceEnabled(true);
    _dataAccessor.updateProperty(_keyBuilder.instanceConfig(instanceName), instanceConfig);
  }
}
 

public static void main(String[] args) throws Exception {
  if (args.length < 3) {
    System.err
        .println("USAGE: java Consumer zookeeperAddress (e.g. localhost:2181) consumerId (0-2), rabbitmqServer (e.g. localhost)");
    System.exit(1);
  }

  final String zkAddr = args[0];
  final String clusterName = SetupConsumerCluster.DEFAULT_CLUSTER_NAME;
  final String consumerId = args[1];
  final String mqServer = args[2];

  ZkClient zkclient = null;
  try {
    // add node to cluster if not already added
    zkclient =
        new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT,
            ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);

    List<String> nodes = admin.getInstancesInCluster(clusterName);
    if (!nodes.contains("consumer_" + consumerId)) {
      InstanceConfig config = new InstanceConfig("consumer_" + consumerId);
      config.setHostName("localhost");
      config.setInstanceEnabled(true);
      admin.addInstance(clusterName, config);
    }

    // start consumer
    final Consumer consumer =
        new Consumer(zkAddr, clusterName, "consumer_" + consumerId, mqServer);

    Runtime.getRuntime().addShutdownHook(new Thread() {
      @Override
      public void run() {
        System.out.println("Shutting down consumer_" + consumerId);
        consumer.disconnect();
      }
    });

    consumer.connect();
  } finally {
    if (zkclient != null) {
      zkclient.close();
    }
  }
}
 

public static void main(String[] args) {
  if (args.length < 2) {
    System.err
        .println("USAGE: java SetupCluster zookeeperAddress(e.g. localhost:2181) numberOfNodes");
    System.exit(1);
  }

  final String zkAddr = args[0];
  final int numNodes = Integer.parseInt(args[1]);
  final String clusterName = DEFAULT_CLUSTER_NAME;

  ZkClient zkclient = null;
  try {
    zkclient =
        new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT,
            ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);

    // add cluster
    admin.addCluster(clusterName, true);

    // add state model definition
    StateModelConfigGenerator generator = new StateModelConfigGenerator();
    admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL,
        new StateModelDefinition(generator.generateConfigForOnlineOffline()));
    // addNodes
    for (int i = 0; i < numNodes; i++) {
      String port = "" + (12001 + i);
      String serverId = "localhost_" + port;
      InstanceConfig config = new InstanceConfig(serverId);
      config.setHostName("localhost");
      config.setPort(port);
      config.setInstanceEnabled(true);
      admin.addInstance(clusterName, config);
    }
    // add resource "repository" which has 1 partition
    String resourceName = DEFAULT_RESOURCE_NAME;
    admin.addResource(clusterName, resourceName, DEFAULT_PARTITION_NUMBER, DEFAULT_STATE_MODEL,
        RebalanceMode.SEMI_AUTO.toString());
    admin.rebalance(clusterName, resourceName, 1);

  } finally {
    if (zkclient != null) {
      zkclient.close();
    }
  }
}