Java Code Examples for org.apache.helix.PropertyType#EXTERNALVIEW

@Test()
public void testUpdateOnNotification() throws Exception {
  MockZkHelixDataAccessor accessor =
      new MockZkHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));

  RoutingDataCache cache =
      new RoutingDataCache("CLUSTER_" + TestHelper.getTestClassName(), PropertyType.EXTERNALVIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);

  accessor.clearReadCounters();

  // refresh again should read nothing
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 0);

  accessor.clearReadCounters();
  // refresh again should read nothing as ideal state is same
  cache.notifyDataChange(HelixConstants.ChangeType.EXTERNAL_VIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 0);
}
 

@Override
public <T extends HelixProperty> List<T> getChildValues(PropertyKey key,
    boolean throwException) {
  PropertyType type = key.getType();
  List<T> result = new ArrayList<T>();
  Class<? extends HelixProperty> clazz = key.getTypeClass();
  if (type == PropertyType.EXTERNALVIEW || type == PropertyType.IDEALSTATES) {
    HelixProperty typedInstance = HelixProperty.convertToTypedInstance(clazz, _externalView);
    result.add((T) typedInstance);
    return result;
  } else if (type == PropertyType.LIVEINSTANCES) {
    return (List<T>) HelixProperty.convertToTypedList(clazz, _liveInstances);
  }

  return result;
}
 

@Test
public void testExternalViewDoesNotExist() {
  String resourceName = WorkflowGenerator.DEFAULT_TGT_DB + 1;
  RoutingTableProvider externalViewProvider =
      new RoutingTableProvider(_manager, PropertyType.EXTERNALVIEW);
  try {
    Assert.assertEquals(externalViewProvider.getInstancesForResource(resourceName, "SLAVE").size(),
        0);
  } finally {
    externalViewProvider.shutdown();
  }
}
 

@Test
public void testMetrics() throws JMException, InterruptedException {
  PropertyType type = PropertyType.EXTERNALVIEW;
  RoutingTableProviderMonitor monitor = new RoutingTableProviderMonitor(type, TEST_CLUSTER);
  monitor.register();
  ObjectName name = buildObjectName(type, TEST_CLUSTER);

  monitor.increaseCallbackCounters(10);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "CallbackCounter"), 1);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "EventQueueSizeGauge"), 10);
  monitor.increaseCallbackCounters(15);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "CallbackCounter"), 2);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "EventQueueSizeGauge"), 15);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "DataRefreshLatencyGauge.Max"), 0);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "DataRefreshCounter"), 0);

  // StatePropagationLatencyGauge only apply for current state
  try {
    _beanServer.getAttribute(name, "StatePropagationLatencyGauge.Max");
    Assert.fail();
  } catch (AttributeNotFoundException ex) {
    // Expected AttributeNotFoundException because the metric does not exist in
    // MBean server.
  }

  long startTime = System.currentTimeMillis();
  Thread.sleep(5);
  monitor.increaseDataRefreshCounters(startTime);
  long latency = (long) _beanServer.getAttribute(name, "DataRefreshLatencyGauge.Max");
  Assert.assertTrue(latency >= 5 && latency <= System.currentTimeMillis() - startTime);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "DataRefreshCounter"), 1);

  monitor.increaseDataRefreshCounters(startTime);
  long newLatency = (long) _beanServer.getAttribute(name, "DataRefreshLatencyGauge.Max");
  Assert.assertTrue(newLatency >= latency);
  Assert.assertEquals((long) _beanServer.getAttribute(name, "DataRefreshCounter"), 2);

  monitor.unregister();
}
 

@Override
public <T extends HelixProperty> T getProperty(PropertyKey key) {

  PropertyType type = key.getType();
  if (type == PropertyType.EXTERNALVIEW || type == PropertyType.IDEALSTATES) {
    return (T) new ExternalView(_externalView);
  }
  return null;
}
 

public RoutingTable(Collection<ExternalView> externalViews,
    Collection<InstanceConfig> instanceConfigs, Collection<LiveInstance> liveInstances) {
  this(externalViews, instanceConfigs, liveInstances,
      PropertyType.EXTERNALVIEW);
}
 

public ExternalViewCache(String clusterName) {
  this(clusterName, PropertyType.EXTERNALVIEW);
}
 

@BeforeClass
public void beforeClass() throws Exception {
  System.out
      .println("START " + getShortClassName() + " at " + new Date(System.currentTimeMillis()));

  // setup storage cluster
  _gSetupTool.addCluster(CLUSTER_NAME, true);

  for (int i = 0; i < PARTICIPANT_NUMBER; i++) {
    String instance = PARTICIPANT_PREFIX + "_" + (PARTICIPANT_START_PORT + i);
    _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instance);
    _instances.add(instance);
  }

  // start dummy participants
  for (int i = 0; i < PARTICIPANT_NUMBER; i++) {
    MockParticipantManager participant =
        new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, _instances.get(i));
    participant.syncStart();
    _participants.add(participant);
  }

  createDBInSemiAuto(_gSetupTool, CLUSTER_NAME, TEST_DB, _instances, STATE_MODEL,
      PARTITION_NUMBER, REPLICA_NUMBER);

  // start controller
  String controllerName = CONTROLLER_PREFIX + "_0";
  _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName);
  _controller.syncStart();

  // start speculator - initialize it with a Mock
  _spectator = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "spectator",
      InstanceType.SPECTATOR, ZK_ADDR);
  _spectator.connect();

  _spectator_2 = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "spectator_2",
      InstanceType.SPECTATOR, ZK_ADDR);
  _spectator_2.connect();

  _spectator_3 = HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "spectator_3",
      InstanceType.SPECTATOR, ZK_ADDR);
  _spectator_3.connect();

  _routingTableProvider =
      new MockRoutingTableProvider(_spectator, PropertyType.EXTERNALVIEW, true, 1000L);
  _spectator.addExternalViewChangeListener(_routingTableProvider);
  _spectator.addLiveInstanceChangeListener(_routingTableProvider);
  _spectator.addInstanceConfigChangeListener(_routingTableProvider);

  _routingTableProviderNoPeriodicRefresh =
      new MockRoutingTableProvider(_spectator_2, PropertyType.EXTERNALVIEW, false, 1000L);
  _spectator_2.addExternalViewChangeListener(_routingTableProviderNoPeriodicRefresh);
  _spectator_2.addLiveInstanceChangeListener(_routingTableProviderNoPeriodicRefresh);
  _spectator_2.addInstanceConfigChangeListener(_routingTableProviderNoPeriodicRefresh);

  _routingTableProviderLongPeriodicRefresh =
      new MockRoutingTableProvider(_spectator_3, PropertyType.EXTERNALVIEW, true, 3000000L);
  _spectator_3.addExternalViewChangeListener(_routingTableProviderLongPeriodicRefresh);
  _spectator_3.addLiveInstanceChangeListener(_routingTableProviderLongPeriodicRefresh);
  _spectator_3.addInstanceConfigChangeListener(_routingTableProviderLongPeriodicRefresh);

  _clusterVerifier =
      new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

}
 

@Test
public void testRoutingTableWithCurrentStates() throws Exception {
  RoutingTableProvider routingTableEV =
      new RoutingTableProvider(_manager, PropertyType.EXTERNALVIEW);
  RoutingTableProvider routingTableCurrentStates =
      new RoutingTableProvider(_manager, PropertyType.CURRENTSTATES);

  try {
    String db1 = "TestDB-1";
    _gSetupTool.addResourceToCluster(CLUSTER_NAME, db1, NUM_PARTITIONS, "MasterSlave",
        IdealState.RebalanceMode.FULL_AUTO.name());
    long startTime = System.currentTimeMillis();
    _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS);

    Thread.sleep(1000L);
    ZkHelixClusterVerifier clusterVerifier =
        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(clusterVerifier.verifyByPolling());
    validatePropagationLatency(PropertyType.CURRENTSTATES,
        System.currentTimeMillis() - startTime);

    IdealState idealState1 =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db1);
    validate(idealState1, routingTableEV, routingTableCurrentStates, 0);

    // add new DB
    String db2 = "TestDB-2";
    _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, NUM_PARTITIONS, "MasterSlave",
        IdealState.RebalanceMode.FULL_AUTO.name());
    startTime = System.currentTimeMillis();
    _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);

    Thread.sleep(1000L);
    Assert.assertTrue(clusterVerifier.verifyByPolling());
    validatePropagationLatency(PropertyType.CURRENTSTATES,
        System.currentTimeMillis() - startTime);

    IdealState idealState2 =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db2);
    validate(idealState2, routingTableEV, routingTableCurrentStates, 0);

    // shutdown an instance
    startTime = System.currentTimeMillis();
    _participants[0].syncStop();
    Thread.sleep(1000L);
    Assert.assertTrue(clusterVerifier.verifyByPolling());
    validatePropagationLatency(PropertyType.CURRENTSTATES,
        System.currentTimeMillis() - startTime);

    validate(idealState1, routingTableEV, routingTableCurrentStates, 0);
    validate(idealState2, routingTableEV, routingTableCurrentStates, 0);
  } finally {
    routingTableEV.shutdown();
    routingTableCurrentStates.shutdown();
  }
}
 

@Test
public void testRoutingTableSnapshot() throws InterruptedException {
  RoutingTableProvider routingTableProvider =
      new RoutingTableProvider(_manager, PropertyType.EXTERNALVIEW);

  try {
    String db1 = "TestDB-1";
    _gSetupTool.addResourceToCluster(CLUSTER_NAME, db1, NUM_PARTITIONS, "MasterSlave",
        IdealState.RebalanceMode.FULL_AUTO.name());
    _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS);

    Thread.sleep(200);
    ZkHelixClusterVerifier clusterVerifier =
        new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(clusterVerifier.verifyByPolling());

    IdealState idealState1 =
        _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db1);

    RoutingTableSnapshot routingTableSnapshot = routingTableProvider.getRoutingTableSnapshot();
    validateMapping(idealState1, routingTableSnapshot);

    Assert.assertEquals(routingTableSnapshot.getInstanceConfigs().size(), NUM_NODES);
    Assert.assertEquals(routingTableSnapshot.getResources().size(), 1);
    Assert.assertEquals(routingTableSnapshot.getLiveInstances().size(), NUM_NODES);
    Assert.assertEquals(routingTableSnapshot.getExternalViews().size(), 1); // 1 DB

    // add new DB and shutdown an instance
    String db2 = "TestDB-2";
    _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, NUM_PARTITIONS, "MasterSlave",
        IdealState.RebalanceMode.FULL_AUTO.name());
    _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);

    // shutdown an instance
    _participants[0].syncStop();
    Thread.sleep(200);
    Assert.assertTrue(clusterVerifier.verifyByPolling());

    // the original snapshot should not change
    Assert.assertEquals(routingTableSnapshot.getInstanceConfigs().size(), NUM_NODES);
    Assert.assertEquals(routingTableSnapshot.getResources().size(), 1);
    Assert.assertEquals(routingTableSnapshot.getLiveInstances().size(), NUM_NODES);

    RoutingTableSnapshot newRoutingTableSnapshot = routingTableProvider.getRoutingTableSnapshot();

    Assert.assertEquals(newRoutingTableSnapshot.getInstanceConfigs().size(), NUM_NODES);
    Assert.assertEquals(newRoutingTableSnapshot.getResources().size(), 2);
    Assert.assertEquals(newRoutingTableSnapshot.getLiveInstances().size(), NUM_NODES - 1);
    Assert.assertEquals(newRoutingTableSnapshot.getExternalViews().size(), 2); // 2 DBs
  } finally {
    routingTableProvider.shutdown();
  }
}
 

@Test (dependsOnMethods = "testTargetExternalViewWithoutEnable")
public void testExternalViewDiffFromTargetExternalView() throws Exception {
  ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
  clusterConfig.enableTargetExternalView(true);
  clusterConfig.setPersistBestPossibleAssignment(true);
  _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
  Thread.sleep(2000);

  RoutingTableProvider externalViewProvider =
      new RoutingTableProvider(_manager, PropertyType.EXTERNALVIEW);
  final RoutingTableProvider targetExternalViewProvider =
      new RoutingTableProvider(_manager, PropertyType.TARGETEXTERNALVIEW);

  try {
    // ExternalView should not contain any MASTERS
    // TargetExternalView should contain MASTERS same as the partition number
    Set<InstanceConfig> externalViewMasters =
        externalViewProvider.getInstancesForResource(WorkflowGenerator.DEFAULT_TGT_DB, "MASTER");
    Assert.assertEquals(externalViewMasters.size(), 0);

    final Set<InstanceConfig> targetExternalViewMasters = new HashSet<>();
    Assert.assertTrue(TestHelper.verify(() -> {
      targetExternalViewMasters.clear();
      targetExternalViewMasters.addAll(targetExternalViewProvider
          .getInstancesForResource(WorkflowGenerator.DEFAULT_TGT_DB, "MASTER"));
      return targetExternalViewMasters.size() == NUM_NODES;
    }, 3000));

    // TargetExternalView MASTERS mapping should exactly match IdealState MASTERS mapping
    Map<String, Map<String, String>> stateMap = _gSetupTool.getClusterManagementTool()
        .getResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB).getRecord().getMapFields();

    Set<String> idealMasters = new HashSet<>();
    Set<String> targetMasters = new HashSet<>();
    for (Map<String, String> instanceMap : stateMap.values()) {
      for (String instance : instanceMap.keySet()) {
        if (instanceMap.get(instance).equals("MASTER")) {
          idealMasters.add(instance);
        }
      }
    }

    for (InstanceConfig instanceConfig : targetExternalViewMasters) {
      targetMasters.add(instanceConfig.getInstanceName());
    }
    Assert.assertEquals(targetMasters, idealMasters);
  } finally {
    externalViewProvider.shutdown();
    targetExternalViewProvider.shutdown();
  }
}
 

@Test(dependsOnMethods = { "testUpdateOnNotification" })
public void testSelectiveUpdates()
    throws Exception {
  MockZkHelixDataAccessor accessor =
      new MockZkHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));

  RoutingDataCache cache =
      new RoutingDataCache("CLUSTER_" + TestHelper.getTestClassName(), PropertyType.EXTERNALVIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);

  accessor.clearReadCounters();

  // refresh again should read nothing
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 0);

  // refresh again should read nothing
  cache.notifyDataChange(HelixConstants.ChangeType.EXTERNAL_VIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 0);

  // add new resources
  _gSetupTool.addResourceToCluster(CLUSTER_NAME, "TestDB_1", 1, STATE_MODEL);
  _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "TestDB_1", _replica);

  Thread.sleep(100);
  ZkHelixClusterVerifier _clusterVerifier =
      new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  accessor.clearReadCounters();

  // refresh again should read only new current states and new idealstate
  cache.notifyDataChange(HelixConstants.ChangeType.EXTERNAL_VIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);

  // Add more resources
  accessor.clearReadCounters();

  _gSetupTool.addResourceToCluster(CLUSTER_NAME, "TestDB_2", 1, STATE_MODEL);
  _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "TestDB_2", _replica);
  _gSetupTool.addResourceToCluster(CLUSTER_NAME, "TestDB_3", 1, STATE_MODEL);
  _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, "TestDB_3", _replica);

  Thread.sleep(100);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  // Totally four resources. Two of them are newly added.
  cache.notifyDataChange(HelixConstants.ChangeType.EXTERNAL_VIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 2);

  // update one resource
  accessor.clearReadCounters();

  _gSetupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, "TestDB_2", false);

  Thread.sleep(100);
  Assert.assertTrue(_clusterVerifier.verifyByPolling());

  cache.notifyDataChange(HelixConstants.ChangeType.EXTERNAL_VIEW);
  cache.refresh(accessor);
  Assert.assertEquals(accessor.getReadCount(PropertyType.EXTERNALVIEW), 1);
}