org.apache.helix.model.ExternalView Java Examples

/**
 * Validate each partition is different instances and with necessary tagged instances.
 */
private void validateIsolation(IdealState is, ExternalView ev, int expectedReplica) {
  String tag = is.getInstanceGroupTag();
  for (String partition : is.getPartitionSet()) {
    Map<String, String> assignmentMap = ev.getRecord().getMapField(partition);
    Set<String> instancesInEV = assignmentMap.keySet();
    Assert.assertEquals(instancesInEV.size(), expectedReplica);
    for (String instance : instancesInEV) {
      if (tag != null) {
        InstanceConfig config =
            _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instance);
        Assert.assertTrue(config.containsTag(tag));
      }
    }
  }
}
 

public static void updateRouteWithValidation(String managerHelixClusterName,
    String routeForHelix,
    String instanceId, ZKHelixAdmin helixAdmin, String state, String expectedState)
    throws InterruptedException {
  if (StringUtils.isBlank(expectedState)) {
    expectedState = state;
  }
  IdealState idealState = TestUtils
      .buildManagerWorkerCustomIdealState(routeForHelix, Collections.singletonList(instanceId),
          state);
  helixAdmin.setResourceIdealState(managerHelixClusterName, routeForHelix, idealState);
  Thread.sleep(1000);
  ExternalView externalView = helixAdmin
      .getResourceExternalView(managerHelixClusterName, routeForHelix);
  Assert.assertNotNull(externalView);
  Assert.assertNotNull(externalView.getStateMap("0"));
  LOGGER.info("ExternalView: {}", externalView);
  Assert.assertEquals(externalView.getStateMap("0").get("0"), expectedState);
}
 

private static JSONObject buildExternalView(String topicName, ExternalView externalViewForTopic) {
  JSONObject externalViewPartitionToServerMappingJson = new JSONObject();
  if (externalViewForTopic == null) {
    LOGGER.info("External view for topic " + topicName + " is NULL");
  } else {
    for (String partition : externalViewForTopic.getPartitionSet()) {
      Map<String, String> stateMap = externalViewForTopic.getStateMap(partition);
      for (String server : stateMap.keySet()) {
        if (!externalViewPartitionToServerMappingJson.containsKey(partition)) {
          externalViewPartitionToServerMappingJson.put(partition, new JSONObject());
        }
        externalViewPartitionToServerMappingJson.getJSONObject(partition).put(server, stateMap.get(server));
      }
    }
  }
  return externalViewPartitionToServerMappingJson;
}
 

@Test (dependsOnMethods = {"testDisableDelayRebalanceInResource"})
public void testDisableDelayRebalanceInCluster() throws Exception {
  enableDelayRebalanceInCluster(_gZkClient, CLUSTER_NAME, true);
  setDelayTimeInCluster(_gZkClient, CLUSTER_NAME, 1000000);
  Map<String, ExternalView> externalViewsBefore = createTestDBs(-1);
  validateDelayedMovements(externalViewsBefore);

  // disable delay rebalance for the entire cluster.
  enableDelayRebalanceInCluster(_gZkClient, CLUSTER_NAME, false);
  Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());
  for (String db : _testDBs) {
    ExternalView ev =
        _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db);
    IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
    validateMinActiveAndTopStateReplica(is, ev, _replica, NUM_NODE);
  }

  enableDelayRebalanceInCluster(_gZkClient, CLUSTER_NAME, true);
}
 

private Set<String> getCurrentPartitionsOnInstance(String cluster, String dbName,
    String instance) {
  HelixAdmin admin = _gSetupTool.getClusterManagementTool();
  Set<String> partitionSet = new HashSet<>();

  ExternalView ev = admin.getResourceExternalView(cluster, dbName);
  for (String partition : ev.getRecord().getMapFields().keySet()) {
    Map<String, String> assignments = ev.getRecord().getMapField(partition);
    for (String ins : assignments.keySet()) {
      if (ins.equals(instance)) {
        partitionSet.add(partition);
      }
    }
  }
  return partitionSet;
}
 

private void getPrintableResourceState(ExternalView resourceExternalView,
                                       String resourceName,
                                       StringBuilder sb) {
  TreeSet<String> sortedSet = new TreeSet<>(resourceExternalView.getPartitionSet());
  sb.append("\nCLUSTER: ");
  sb.append(getClusterName());
  sb.append("\nRESOURCE: ");
  sb.append(resourceName);
  for (String partitionName : sortedSet) {
    sb.append("\nPARTITION: ");
    sb.append(partitionName).append("\t");
    Map<String, String> states = resourceExternalView.getStateMap(partitionName);
    for (Map.Entry<String, String> stateEntry : states.entrySet()) {
      sb.append("\t");
      sb.append(stateEntry.getKey());
      sb.append("\t");
      sb.append(stateEntry.getValue());
    }
  }
}
 

private ExternalView prepareExternalView() {
  ExternalView externalView = new ExternalView(RESOURCE_NAME);
  externalView.getRecord()
      .setSimpleField(ExternalView.ExternalViewProperty.STATE_MODEL_DEF_REF.toString(),
          MasterSlaveSMD.name);
  externalView.setState("p1", "h1", "MASTER");
  externalView.setState("p1", "h2", "SLAVE");
  externalView.setState("p1", "h3", "SLAVE");

  externalView.setState("p2", "h1", "SLAVE");
  externalView.setState("p2", "h2", "MASTER");
  externalView.setState("p2", "h3", "SLAVE");

  externalView.setState("p3", "h1", "SLAVE");
  externalView.setState("p3", "h2", "MASTER");
  externalView.setState("p3", "h3", "SLAVE");

  return externalView;
}
 

HelixAdmin makeHelixAdmin() {
  HelixAdmin admin = mock(HelixAdmin.class);
  ExternalView ev = mock(ExternalView.class);
  IdealState is = mock(IdealState.class);
  when(admin.getResourceExternalView(clusterName, tableName)).thenReturn(ev);
  when(admin.getResourceIdealState(clusterName, tableName)).thenReturn(is);

  List<String> segmentsInIs = segmentsInIdealStateOrExtView();
  Map<String, String> dummy = new HashMap<>(1);
  dummy.put("someHost", "ONLINE");
  for (String segment : segmentsInIs) {
    when(is.getInstanceStateMap(segment)).thenReturn(dummy);
  }
  when(ev.getStateMap(anyString())).thenReturn(null);

  return admin;
}
 

@Test
public void TestSiblingNodesActiveReplicaCheck_whenNoMinActiveReplica() {
  String resource = "resource";
  Mock mock = new Mock();
  doReturn(ImmutableList.of(resource)).when(mock.dataAccessor)
      .getChildNames(argThat(new PropertyKeyArgument(PropertyType.IDEALSTATES)));
  // set ideal state
  IdealState idealState = mock(IdealState.class);
  when(idealState.isEnabled()).thenReturn(true);
  when(idealState.isValid()).thenReturn(true);
  when(idealState.getStateModelDefRef()).thenReturn("MasterSlave");
  doReturn(idealState).when(mock.dataAccessor).getProperty(argThat(new PropertyKeyArgument(PropertyType.IDEALSTATES)));
  //set externalView
  ExternalView externalView = mock(ExternalView.class);
  // the resource sibling check will be skipped by design
  when(externalView.getMinActiveReplicas()).thenReturn(-1);
  doReturn(externalView).when(mock.dataAccessor)
      .getProperty(argThat(new PropertyKeyArgument(PropertyType.EXTERNALVIEW)));

  boolean result = InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE);
  Assert.assertTrue(result);
}
 

@Test
public void testOfflineTableNotnullQuota()
    throws Exception {
  ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME);
  TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME);
  ZKMetadataProvider
      .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfigUtils.toZNRecord(tableConfig));
  setQps(tableConfig);
  _queryQuotaManager.initTableQueryQuota(tableConfig, brokerResource);
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 1);

  // All the request should be passed.
  runQueries(70, 10);

  _queryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME);
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 0);
}
 

private static void printState(String msg) {
  System.out.println("CLUSTER STATE: " + msg);
  ExternalView resourceExternalView = admin.getResourceExternalView(CLUSTER_NAME, RESOURCE_NAME);
  TreeSet<String> sortedSet = new TreeSet<String>(resourceExternalView.getPartitionSet());
  StringBuilder sb = new StringBuilder("\t\t");
  for (int i = 0; i < NUM_NODES; i++) {
    sb.append(INSTANCE_CONFIG_LIST.get(i).getInstanceName()).append("\t");
  }
  System.out.println(sb);
  for (String partitionName : sortedSet) {
    sb.delete(0, sb.length() - 1);
    sb.append(partitionName).append("\t");
    for (int i = 0; i < NUM_NODES; i++) {
      Map<String, String> stateMap = resourceExternalView.getStateMap(partitionName);
      if (stateMap != null && stateMap.containsKey(INSTANCE_CONFIG_LIST.get(i).getInstanceName())) {
        sb.append(stateMap.get(INSTANCE_CONFIG_LIST.get(i).getInstanceName()).charAt(0)).append(
            "\t\t");
      } else {
        sb.append("-").append("\t\t");
      }
    }
    System.out.println(sb);
  }
  System.out.println("###################################################################");
}
 

@Test
public void testPartitionLevelCheck() {
  List<ExternalView> externalViews = new ArrayList<>(Arrays.asList(prepareExternalView()));
  Mock mock = new Mock();
  HelixDataAccessor accessor = mock.dataAccessor;

  when(mock.dataAccessor.keyBuilder())
      .thenReturn(new PropertyKey.Builder(TEST_CLUSTER));
  when(mock.dataAccessor
      .getProperty(new PropertyKey.Builder(TEST_CLUSTER).stateModelDef(MasterSlaveSMD.name)))
      .thenReturn(mock.stateModel);
  when(mock.stateModel.getTopState()).thenReturn("MASTER");
  List<String> failedPartitions = InstanceValidationUtil
      .perPartitionHealthCheck(externalViews, preparePartitionStateMap(), "h2", accessor);

  Assert.assertTrue(failedPartitions.size() == 1);
  Assert.assertEquals(failedPartitions.iterator().next(), "p2");
}
 

@Test
public void testCancellationWhenDisableResource() throws InterruptedException {
  // Enable cancellation
  ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
  clusterConfig.stateTransitionCancelEnabled(true);
  _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);

  // Wait for assignment done
  Thread.sleep(2000);

  // Disable the resource
  _gSetupTool.getClusterManagementTool().enableResource(CLUSTER_NAME,
      WorkflowGenerator.DEFAULT_TGT_DB, false);

  // Wait for pipeline reaching final stage
  Assert.assertTrue(_verifier.verifyByPolling());
  ExternalView externalView = _gSetupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  for (String partition : externalView.getPartitionSet()) {
    for (String currentState : externalView.getStateMap(partition).values()) {
      Assert.assertEquals(currentState, "OFFLINE");
    }
  }
}
 

@Test(dependsOnMethods = "testInMaintenanceMode")
public void testMaintenanceModeAddNewInstance() {
  _gSetupTool.getClusterManagementTool().enableMaintenanceMode(CLUSTER_NAME, true, "Test");
  ExternalView prevExternalView = _gSetupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  String instanceName = PARTICIPANT_PREFIX + "_" + (_startPort + 10);
  _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instanceName);
  _newInstance = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName);
  _newInstance.syncStart();
  _gSetupTool.getClusterManagementTool().rebalance(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB,
      3);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());
  ExternalView newExternalView = _gSetupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  Assert.assertEquals(prevExternalView.getRecord().getMapFields(),
      newExternalView.getRecord().getMapFields());
}
 

private void testHourlyPushTable(String rawTableName, TableConfig tableConfig, TimeUnit timeUnit, ExternalView externalView) {
  TimeBoundaryManager timeBoundaryManager = new TimeBoundaryManager(tableConfig, _propertyStore);
  Set<String> onlineSegments = new HashSet<>();
  String segment0 = "segment0";
  onlineSegments.add(segment0);
  setSegmentZKMetadata(rawTableName, segment0, 2, timeUnit);
  timeBoundaryManager.init(externalView, onlineSegments);
  long expectedTimeValue;
  if (timeUnit == TimeUnit.DAYS) {
    // Time boundary should be endTime - 1 DAY when time unit is DAYS
    expectedTimeValue = timeUnit.convert(1, TimeUnit.DAYS);
  } else {
    // Time boundary should be endTime - 1 HOUR when time unit is other than DAYS
    expectedTimeValue = timeUnit.convert(47, TimeUnit.HOURS);
  }
  verifyTimeBoundaryInfo(timeBoundaryManager.getTimeBoundaryInfo(), expectedTimeValue);
}
 

private static void printStatus(HelixAdmin admin, String cluster, String resource) {
  ExternalView externalView = admin.getResourceExternalView(cluster, resource);
  // System.out.println(externalView);
  TreeSet<String> treeSet = new TreeSet<String>(externalView.getPartitionSet());
  System.out.println("lockName" + "\t" + "acquired By");
  System.out.println("======================================");
  for (String lockName : treeSet) {
    Map<String, String> stateMap = externalView.getStateMap(lockName);
    String acquiredBy = null;
    if (stateMap != null) {
      for (String instanceName : stateMap.keySet()) {
        if ("ONLINE".equals(stateMap.get(instanceName))) {
          acquiredBy = instanceName;
          break;
        }
      }
    }
    System.out.println(lockName + "\t" + ((acquiredBy != null) ? acquiredBy : "NONE"));
  }
}
 

@Test(enabled = false)
public void testOldDisableBatchEnable() throws InterruptedException {
  _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME,
      _participants[0].getInstanceName(), false);
  _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME,
      Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()),
      true);
  Thread.sleep(2000);

  ExternalView externalView = _gSetupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions);
  int numOfFirstHost = 0;
  for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) {
    if (stateMap.keySet().contains(_participants[0].getInstanceName())) {
      numOfFirstHost++;
    }
  }
  Assert.assertTrue(numOfFirstHost > 0);
  _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME,
      _participants[0].getInstanceName(), true);
}
 

@Test(enabled = false)
public void testBatchEnableDisable() throws InterruptedException {
  _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME,
      Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()),
      false);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  ExternalView externalView = _gSetupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions);
  for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) {
    Assert.assertTrue(!stateMap.keySet().contains(_participants[0].getInstanceName()));
    Assert.assertTrue(!stateMap.keySet().contains(_participants[1].getInstanceName()));
  }
  _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME,
      Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()),
      true);
}
 

/**
 * Test when two nodes go offline,  the minimal active replica should be maintained.
 * @throws Exception
 */
@Test(dependsOnMethods = {"testDelayedPartitionMovement"})
public void testMinimalActiveReplicaMaintain() throws Exception {
  setDelayTimeInCluster(_gZkClient, CLUSTER_NAME, 1000000);
  Map<String, ExternalView> externalViewsBefore = createTestDBs(-1);
  validateDelayedMovements(externalViewsBefore);

  // bring down another node, the minimal active replica for each partition should be maintained.
  _participants.get(3).syncStop();
  Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());
  for (String db : _testDBs) {
    ExternalView ev =
        _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db);
    IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
    validateMinActiveAndTopStateReplica(is, ev, _minActiveReplica, NUM_NODE);
  }
  setDelayTimeInCluster(_gZkClient, CLUSTER_NAME, -1);
}
 

/**
 * Assert externalView and currentState for each participant are empty
 * @param clusterName
 * @param db
 * @param participants
 */
private void assertEmptyCSandEV(String clusterName, String db,
    MockParticipantManager[] participants) throws Exception {
  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_gZkClient));
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  boolean isExternalViewNull = TestHelper.verify(() -> {
    ExternalView externalView = accessor.getProperty(keyBuilder.externalView(db));
    return (externalView == null);
  }, TestHelper.WAIT_DURATION);
  Assert.assertTrue(isExternalViewNull);

  for (MockParticipantManager participant : participants) {
    String instanceName = participant.getInstanceName();
    String sessionId = participant.getSessionId();
    boolean isCurrentStateNull = TestHelper.verify(() -> {
      CurrentState currentState = accessor.getProperty(keyBuilder.currentState(instanceName, sessionId, db));
      return (currentState == null);
    }, TestHelper.WAIT_DURATION);
    Assert.assertTrue(isCurrentStateNull);
  }
}
 

private Map<String, StoppableCheck> performPartitionsCheck(List<String> instances,
    RESTConfig restConfig, Map<String, String> customPayLoads) {
  Map<String, Map<String, Boolean>> allPartitionsHealthOnLiveInstance =
      _dataAccessor.getAllPartitionsHealthOnLiveInstance(restConfig, customPayLoads);
  List<ExternalView> externalViews =
      _dataAccessor.getChildValues(_dataAccessor.keyBuilder().externalViews(), true);
  Map<String, StoppableCheck> instanceStoppableChecks = new HashMap<>();
  for (String instanceName : instances) {
    List<String> unHealthyPartitions = InstanceValidationUtil
        .perPartitionHealthCheck(externalViews, allPartitionsHealthOnLiveInstance, instanceName,
            _dataAccessor);
    StoppableCheck stoppableCheck =
        new StoppableCheck(unHealthyPartitions.isEmpty(), unHealthyPartitions,
            StoppableCheck.Category.CUSTOM_PARTITION_CHECK);
    instanceStoppableChecks.put(instanceName, stoppableCheck);
  }

  return instanceStoppableChecks;
}
 

@Test (dependsOnMethods = {"testDisableDelayRebalanceInCluster"})
public void testDisableDelayRebalanceInInstance() throws Exception {
  setDelayTimeInCluster(_gZkClient, CLUSTER_NAME, 1000000);
  Map<String, ExternalView> externalViewsBefore = createTestDBs(-1);
  validateDelayedMovements(externalViewsBefore);

  String disabledInstanceName = _participants.get(0).getInstanceName();
  enableDelayRebalanceInInstance(_gZkClient, CLUSTER_NAME, disabledInstanceName, false);
  Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());
  for (String db : _testDBs) {
    IdealState is = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
    Map<String, List<String>> preferenceLists = is.getPreferenceLists();
    for (List<String> instances : preferenceLists.values()) {
      Assert.assertFalse(instances.contains(disabledInstanceName));
    }
  }
  enableDelayRebalanceInInstance(_gZkClient, CLUSTER_NAME, disabledInstanceName, true);
}
 

private long getTopStateUpdateTime(ExternalView ev, String partition, String state) {
  String topStateHost = ev.getStateMap(partition).entrySet().stream()
      .filter(entry -> entry.getValue().equals(state)).findFirst().get().getKey();
  MockParticipantManager participant = Arrays.stream(_participants)
      .filter(instance -> instance.getInstanceName().equals(topStateHost)).findFirst().get();

  HelixDataAccessor accessor = _controller.getHelixDataAccessor();
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  CurrentState currentState = accessor.getProperty(keyBuilder
      .currentState(participant.getInstanceName(), participant.getSessionId(),
          ev.getResourceName()));
  return currentState.getEndTime(partition);
}
 

public void printClusterState() {
  StringBuilder sb = new StringBuilder("\n######################### Cluster HA state ########################");

  ExternalView resourceExternalView = admin.getResourceExternalView(getClusterName(), METRIC_AGGREGATORS);
  if (resourceExternalView != null) {
    getPrintableResourceState(resourceExternalView, METRIC_AGGREGATORS, sb);
  }
  sb.append("\n##################################################");
  LOG.info(sb.toString());
}
 

private boolean verifyExternalView(ResourceControllerDataProvider dataCache, ExternalView externalView,
    IdealState idealState) {
  Map<String, Map<String, String>> mappingInExtview = externalView.getRecord().getMapFields();
  Map<String, Map<String, String>> idealPartitionState;

  switch (idealState.getRebalanceMode()) {
  case FULL_AUTO:
    ClusterConfig clusterConfig = new ConfigAccessor(_zkClient).getClusterConfig(dataCache.getClusterName());
    if (!clusterConfig.isPersistBestPossibleAssignment() && !clusterConfig.isPersistIntermediateAssignment()) {
      throw new HelixException(String.format("Full-Auto IdealState verifier requires "
          + "ClusterConfig.PERSIST_BEST_POSSIBLE_ASSIGNMENT or ClusterConfig.PERSIST_INTERMEDIATE_ASSIGNMENT "
          + "is enabled."));
    }
    for (String partition : idealState.getPartitionSet()) {
      if (idealState.getPreferenceList(partition) == null || idealState.getPreferenceList(partition).isEmpty()) {
        return false;
      }
    }
    idealPartitionState = computeIdealPartitionState(dataCache, idealState);
    break;
  case SEMI_AUTO:
  case USER_DEFINED:
    idealPartitionState = computeIdealPartitionState(dataCache, idealState);
    break;
  case CUSTOMIZED:
    idealPartitionState = idealState.getRecord().getMapFields();
    break;
  case TASK:
    // ignore jobs
  default:
    return true;
  }

  return mappingInExtview.equals(idealPartitionState);
}
 

private boolean verifyExternalView(ExternalView externalView,
    PartitionStateMap bestPossibleState, StateModelDefinition stateModelDef) {
  Set<String> ignoreStates = new HashSet<>(
      Arrays.asList(stateModelDef.getInitialState(), HelixDefinedState.DROPPED.toString()));

  Map<String, Map<String, String>> bestPossibleStateMap =
      convertBestPossibleState(bestPossibleState);

  removeEntryWithIgnoredStates(bestPossibleStateMap.entrySet().iterator(), ignoreStates);

  Map<String, Map<String, String>> externalViewMap = externalView.getRecord().getMapFields();
  removeEntryWithIgnoredStates(externalViewMap.entrySet().iterator(), ignoreStates);

  return externalViewMap.equals(bestPossibleStateMap);
}
 

@Test
public void testOfflineToSecondTopState() throws Exception {
  _participants[0].syncStop();
  Thread.sleep(2000L);

  ExternalView externalView =
      _accessor.getProperty(_keyBuilder.externalView(WorkflowGenerator.DEFAULT_TGT_DB + "0"));
  Map<String, String> stateMap =
      externalView.getStateMap(WorkflowGenerator.DEFAULT_TGT_DB + "0_0");
  Assert.assertEquals("MASTER", stateMap.get(PARTICIPANT_PREFIX + "_" + (_startPort + 1)));
  Assert.assertEquals("SLAVE", stateMap.get(PARTICIPANT_PREFIX + "_" + (_startPort + 2)));

  String instanceName = PARTICIPANT_PREFIX + "_" + _startPort;
  _participants[0] = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName);

  // add a state model with non-OFFLINE initial state
  StateMachineEngine stateMach = _participants[0].getStateMachineEngine();
  MockDelayMSStateModelFactory delayFactory =
      new MockDelayMSStateModelFactory().setDelay(300000L);
  stateMach.registerStateModelFactory(MASTER_SLAVE_STATE_MODEL, delayFactory);

  _participants[0].syncStart();
  Thread.sleep(2000L);

  externalView =
      _accessor.getProperty(_keyBuilder.externalView(WorkflowGenerator.DEFAULT_TGT_DB + "0"));
  stateMap = externalView.getStateMap(WorkflowGenerator.DEFAULT_TGT_DB + "0_0");
  Assert.assertEquals("OFFLINE", stateMap.get(PARTICIPANT_PREFIX + "_" + _startPort));
  Assert.assertEquals("MASTER", stateMap.get(PARTICIPANT_PREFIX + "_" + (_startPort + 1)));
  Assert.assertEquals("SLAVE", stateMap.get(PARTICIPANT_PREFIX + "_" + (_startPort + 2)));
}
 

private void validate(int expectedReplica) {
  ZkHelixClusterVerifier _clusterVerifier =
      new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  for (String db : _allDBs) {
    IdealState is =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
    ExternalView ev =
        _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db);
    validateZoneAndTagIsolation(is, ev, expectedReplica);
  }
}
 

public static boolean verifyState(RealmAwareZkClient zkclient, String clusterName, String resourceName,
    Map<String, Map<String, String>> expectStateMap, String op) {
  boolean result = true;
  ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(zkclient);
  ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
  Builder keyBuilder = accessor.keyBuilder();

  ExternalView extView = accessor.getProperty(keyBuilder.externalView(resourceName));
  Map<String, Map<String, String>> actualStateMap = extView.getRecord().getMapFields();
  for (String partition : actualStateMap.keySet()) {
    for (String expectPartiton : expectStateMap.keySet()) {
      if (!partition.matches(expectPartiton)) {
        continue;
      }

      Map<String, String> actualInstanceStateMap = actualStateMap.get(partition);
      Map<String, String> expectInstanceStateMap = expectStateMap.get(expectPartiton);
      for (String instance : actualInstanceStateMap.keySet()) {
        for (String expectInstance : expectStateMap.get(expectPartiton).keySet()) {
          if (!instance.matches(expectInstance)) {
            continue;
          }

          String actualState = actualInstanceStateMap.get(instance);
          String expectState = expectInstanceStateMap.get(expectInstance);
          boolean equals = expectState.equals(actualState);
          if (op.equals("==") && !equals || op.equals("!=") && equals) {
            System.out.println(
                partition + "/" + instance + " state mismatch. actual state: " + actualState
                    + ", but expect: " + expectState + ", op: " + op);
            result = false;
          }
        }
      }
    }
  }
  return result;
}
 

private void validate(int expectedReplica) {
  HelixClusterVerifier _clusterVerifier =
      new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
          .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
  Assert.assertTrue(_clusterVerifier.verify(5000));
  for (String db : _allDBs) {
    IdealState is =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
    ExternalView ev =
        _gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db);
    validateIsolation(is, ev, expectedReplica);
  }
}