Java Code Examples for org.apache.helix.model.IdealState#setMinActiveReplicas()

private IdealState createResource(String clusterName, String db, String stateModel,
    int numPartition, int replica, int minActiveReplica, long delay, String rebalancerClassName,
    String rebalanceStrategy) {
  IdealState idealState =
      _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db);
  if (idealState == null) {
    _gSetupTool.addResourceToCluster(clusterName, db, numPartition, stateModel,
        IdealState.RebalanceMode.FULL_AUTO + "", rebalanceStrategy);
  }

  idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db);
  idealState.setMinActiveReplicas(minActiveReplica);
  if (!idealState.isDelayRebalanceEnabled()) {
    idealState.setDelayRebalanceEnabled(true);
  }
  if (delay > 0) {
    idealState.setRebalanceDelay(delay);
  }
  idealState.setRebalancerClassName(rebalancerClassName);
  _gSetupTool.getClusterManagementTool().setResourceIdealState(clusterName, db, idealState);
  _gSetupTool.rebalanceStorageCluster(clusterName, db, replica);
  idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(clusterName, db);

  return idealState;
}
 

/**
 * Set up delayed rebalancer and minimum active replica settings to mimic user's use case.
 * @param clusterName
 * @param participantCount
 * @throws Exception
 */
private void setupCluster(String clusterName, int participantCount) throws Exception {
  TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant start port
      "localhost", // participant name prefix
      _resourceName, // resource name prefix
      3, // resources
      5, // partitions per resource
      participantCount, // number of nodes
      3, // replicas
      "MasterSlave", IdealState.RebalanceMode.FULL_AUTO, true); // do rebalance

  // Enable DelayedAutoRebalance
  ClusterConfig clusterConfig = _accessor.getProperty(_accessor.keyBuilder().clusterConfig());
  clusterConfig.setDelayRebalaceEnabled(true);
  clusterConfig.setRebalanceDelayTime(1800000L);
  _accessor.setProperty(_accessor.keyBuilder().clusterConfig(), clusterConfig);

  // Set minActiveReplicas at 2
  List<String> idealStates = _accessor.getChildNames(_accessor.keyBuilder().idealStates());
  for (String is : idealStates) {
    IdealState idealState = _accessor.getProperty(_accessor.keyBuilder().idealStates(is));
    idealState.setMinActiveReplicas(2);
    idealState.setRebalanceStrategy(
        "org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy");
    idealState
        .setRebalancerClassName("org.apache.helix.controller.rebalancer.DelayedAutoRebalancer");
    _accessor.setProperty(_accessor.keyBuilder().idealStates(is), idealState);
  }
}
 

protected List<IdealState> setupIdealState(int nodes, String[] resources, int partitions,
    int replicas, RebalanceMode rebalanceMode, String stateModelName, String rebalanceClassName,
    String rebalanceStrategyName, int minActiveReplica) {
  List<IdealState> idealStates = new ArrayList<IdealState>();
  for (String resourceName : resources) {
    ZNRecord record = new ZNRecord(resourceName);
    for (int p = 0; p < partitions; p++) {
      List<String> value = new ArrayList<String>();
      for (int r = 0; r < replicas; r++) {
        value.add(HOSTNAME_PREFIX + (p + r + 1) % nodes);
      }
      record.setListField(resourceName + "_" + p, value);
    }
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef(stateModelName);
    if (rebalanceClassName != null) {
      idealState.setRebalancerClassName(rebalanceClassName);
    }
    if (rebalanceStrategyName != null) {
      idealState.setRebalanceStrategy(rebalanceStrategyName);
    }
    idealState.setRebalanceMode(rebalanceMode);
    idealState.setNumPartitions(partitions);
    idealStates.add(idealState);
    idealState.setReplicas(String.valueOf(replicas));

    if (minActiveReplica > 0) {
      idealState.setMinActiveReplicas(minActiveReplica);
    }

    Builder keyBuilder = accessor.keyBuilder();

    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
  }
  return idealStates;
}
 

protected Set<String> createResources(String cluster, int numResources) {
  Set<String> resources = new HashSet<>();
  for (int i = 0; i < numResources; i++) {
    String resource = cluster + "_db_" + i;
    _gSetupTool.addResourceToCluster(cluster, resource, NUM_PARTITIONS, "MasterSlave");
    IdealState idealState =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(cluster, resource);
    idealState.setMinActiveReplicas(MIN_ACTIVE_REPLICA);
    _gSetupTool.getClusterManagementTool().setResourceIdealState(cluster, resource, idealState);
    _gSetupTool.rebalanceStorageCluster(cluster, resource, NUM_REPLICA);
    resources.add(resource);
  }
  return resources;
}
 

/**
 * @return
 */
public IdealState build() {
  IdealState idealstate = new IdealState(_record);
  idealstate.setNumPartitions(numPartitions);
  idealstate.setStateModelDefRef(stateModel);
  idealstate.setStateModelFactoryName(stateModelFactoryName);
  idealstate.setRebalanceMode(rebalancerMode);
  idealstate.setReplicas("" + numReplica);

  if (minActiveReplica >= 0) {
    idealstate.setMinActiveReplicas(minActiveReplica);
  }

  if (rebalancerClassName != null) {
    idealstate.setRebalancerClassName(rebalancerClassName);
  }

  if (rebalanceStrategy != null) {
    idealstate.setRebalanceStrategy(rebalanceStrategy);
  }

  if (maxPartitionsPerNode > 0) {
    idealstate.setMaxPartitionsPerInstance(maxPartitionsPerNode);
  }

  if (disableExternalView != null) {
    idealstate.setDisableExternalView(disableExternalView);
  }

  if (enableGroupRouting != null) {
    idealstate.enableGroupRouting(enableGroupRouting);
  }

  if (resourceGroupName != null) {
    idealstate.setResourceGroupName(resourceGroupName);
  }

  if (resourceType != null) {
    idealstate.setResourceType(resourceType);
  }

  if (rebalanceDelayInMs >= 0) {
    idealstate.setRebalanceDelay(rebalanceDelayInMs);
  }

  if (delayRebalanceEnabled != null) {
    idealstate.setDelayRebalanceEnabled(delayRebalanceEnabled);
  }

  if (!idealstate.isValid()) {
    throw new HelixException("invalid ideal-state: " + idealstate);
  }
  return idealstate;
}
 

@Test(dependsOnMethods = {
    "testResourceThrottle"
})
public void testResourceThrottleWithDelayRebalancer() {
  // start a few participants
  for (int i = 0; i < NODE_NR - 2; i++) {
    _participants[i].syncStart();
  }

  int partition = 10;
  int replica = 3;
  int minActiveReplica = 2;
  int delay = 100;

  for (int i = 0; i < 5; i++) {
    String dbName = "TestDB-" + i;
    IdealState is =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, dbName);
    if (is != null) {
      System.err.println(dbName + "exists!");
      is.setReplicas(String.valueOf(replica));
      is.setMinActiveReplicas(minActiveReplica);
      is.setRebalanceDelay(delay);
      is.setRebalancerClassName(DelayedAutoRebalancer.class.getName());
      _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, dbName, is);
    } else {
      createResourceWithDelayedRebalance(CLUSTER_NAME, dbName, STATE_MODEL, partition, replica,
          minActiveReplica, delay);
    }
    _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, dbName, _replica);
    _dbs.add(dbName);
  }

  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  DelayedTransition.setDelay(20);
  DelayedTransition.enableThrottleRecord();

  // add 2 nodes
  for (int i = NODE_NR - 2; i < NODE_NR; i++) {
    _participants[i].syncStart();
  }

  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  for (String db : _dbs) {
    int maxInParallel =
        getMaxParallelTransitionCount(DelayedTransition.getResourcePatitionTransitionTimes(), db);
    System.out.println("MaxInParallel: " + maxInParallel + " maxPendingTransition: " + 2);
    Assert.assertTrue(maxInParallel <= 2, "Throttle condition does not meet for " + db);
  }
}
 

/**
 * Creates a setup where the health API can be tested.
 * @param clusterName
 * @param resourceName
 * @param idealStateParams
 * @param partitionReplicaStates maps partitionName to its replicas' states
 * @throws Exception
 */
private void createDummyMapping(String clusterName, String resourceName,
    Map<String, String> idealStateParams, Map<String, List<String>> partitionReplicaStates)
    throws Exception {
  IdealState idealState = new IdealState(resourceName);
  idealState.setMinActiveReplicas(Integer.parseInt(idealStateParams.get("MinActiveReplicas"))); // 2
  idealState.setStateModelDefRef(idealStateParams.get("StateModelDefRef")); // MasterSlave
  idealState.setMaxPartitionsPerInstance(
      Integer.parseInt(idealStateParams.get("MaxPartitionsPerInstance"))); // 3
  idealState.setReplicas(idealStateParams.get("Replicas")); // 3
  idealState.setNumPartitions(Integer.parseInt(idealStateParams.get("NumPartitions"))); // 3
  idealState.enable(false);

  Map<String, List<String>> partitionNames = new LinkedHashMap<>();
  List<String> dummyPrefList = new ArrayList<>();

  for (int i = 0; i < Integer.parseInt(idealStateParams.get("MaxPartitionsPerInstance")); i++) {
    dummyPrefList.add(ANY_INSTANCE);
    partitionNames.put("p" + i, dummyPrefList);
  }
  idealState.getRecord().getListFields().putAll(partitionNames);

  if (!_gSetupTool.getClusterManagementTool().getClusters().contains(clusterName)) {
    _gSetupTool.getClusterManagementTool().addCluster(clusterName);
  }
  _gSetupTool.getClusterManagementTool().setResourceIdealState(clusterName, resourceName,
      idealState);

  // Set ExternalView's replica states for a given parameter map
  ExternalView externalView = new ExternalView(resourceName);

  Map<String, Map<String, String>> mappingCurrent = new LinkedHashMap<>();

  List<String> partitionReplicaStatesList = new ArrayList<>(partitionReplicaStates.keySet());
  for (int k = 0; k < partitionReplicaStatesList.size(); k++) {
    Map<String, String> replicaStatesForPartition = new LinkedHashMap<>();
    List<String> replicaStateList = partitionReplicaStates.get(partitionReplicaStatesList.get(k));
    for (int i = 0; i < replicaStateList.size(); i++) {
      replicaStatesForPartition.put("r" + i, replicaStateList.get(i));
    }
    mappingCurrent.put("p" + k, replicaStatesForPartition);
  }

  externalView.getRecord().getMapFields().putAll(mappingCurrent);

  HelixManager helixManager = HelixManagerFactory.getZKHelixManager(clusterName, "p1",
      InstanceType.ADMINISTRATOR, ZK_ADDR);
  helixManager.connect();
  HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor();
  helixDataAccessor.setProperty(helixDataAccessor.keyBuilder().externalView(resourceName),
      externalView);
  System.out.println("End test :" + TestHelper.getTestMethodName());
}