Java Code Examples for org.apache.helix.HelixDataAccessor#getProperty()

@GET @Path("resources/{resourceName}")
public Response getResourceOnInstance(@PathParam("clusterId") String clusterId,
    @PathParam("instanceName") String instanceName,
    @PathParam("resourceName") String resourceName) throws IOException {
  HelixDataAccessor accessor = getDataAccssor(clusterId);
  List<String> sessionIds = accessor.getChildNames(accessor.keyBuilder().sessions(instanceName));
  if (sessionIds == null || sessionIds.size() == 0) {
    return notFound();
  }

  // Only get resource list from current session id
  String currentSessionId = sessionIds.get(0);
  CurrentState resourceCurrentState = accessor.getProperty(
      accessor.keyBuilder().currentState(instanceName, currentSessionId, resourceName));
  if (resourceCurrentState != null) {
    return JSONRepresentation(resourceCurrentState.getRecord());
  }

  return notFound();
}
 

private boolean verify(HelixDataAccessor accessor, IdealState idealState,
    Map<String, SleepStateModelFactory> factoryMap) {
  Builder kb = accessor.keyBuilder();
  ExternalView ev = accessor.getProperty(kb.externalView(TEST_DB));
  for (String p : idealState.getPartitionSet()) {
    String idealMaster = idealState.getPreferenceList(p).get(0);
    if (!ev.getStateMap(p).get(idealMaster).equals("ERROR")) {
      return false;
    }

    TimeOutStateModel model = factoryMap.get(idealMaster).getStateModel(TEST_DB, p);
    if (model._errorCallcount != 1 || model._error.getCode() != ErrorCode.TIMEOUT) {
      return false;
    }
  }

  return true;
}
 

private List<HelixProperty> getProperty(HelixDataAccessor accessor, String dataSpec,
    PropertyKey propertyKeys, PropertyKey propertyKey, String dataType) {
  List<HelixProperty> properties;
  if (Strings.isNullOrEmpty(dataSpec) || dataSpec.equals(MATCH_ALL_SYM)) {
    // TODO: Apply strict check on the getChildValues() call.
    // TODO: For backward compatibility, allow partial read for now. This may reduce the
    // TODO: matches eventually.
    properties = accessor.getChildValues(propertyKeys, false);
  } else {
    HelixProperty data = accessor.getProperty(propertyKey);
    if (data == null) {
      throw new HelixException(
          String.format("Specified %s %s is not found!", dataType, dataSpec));
    }
    properties = Collections.singletonList(data);
  }
  return properties;
}
 

private void acquireLeadership(final HelixManager manager,
    ControllerManagerHelper controllerHelper) {
  HelixDataAccessor accessor = manager.getHelixDataAccessor();
  PropertyKey leaderNodePropertyKey = accessor.keyBuilder().controllerLeader();

  LOG.info(manager.getInstanceName() + " tries to acquire leadership for cluster: " + manager
      .getClusterName());
  // Try to acquire leader and init the manager in any case.
  // Even when a leader node already exists, the election process shall still try to init the manager
  // in case it is the current leader.
  do {
    // Due to the possible carried over ZK events from the previous ZK session, the following
    // initialization might be triggered multiple times. So the operation must be idempotent.
    long start = System.currentTimeMillis();
    if (tryCreateController(manager)) {
      manager.getHelixDataAccessor().getBaseDataAccessor().reset();
      controllerHelper.addListenersToController(_controller);
      controllerHelper.startControllerTimerTasks();
      LOG.info("{} with session {} acquired leadership for cluster: {}, took: {}ms",
          manager.getInstanceName(), manager.getSessionId(), manager.getClusterName(),
          System.currentTimeMillis() - start);
    }
  } while (accessor.getProperty(leaderNodePropertyKey) == null);
}
 

@Override
public void setInstanceZoneId(String clusterName, String instanceName, String zoneId) {
  logger.info("Set instance zoneId {} for instance {} in cluster {}.", zoneId, instanceName,
      clusterName);
  if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
    throw new HelixException("cluster " + clusterName + " is not setup yet");
  }

  if (!ZKUtil.isInstanceSetup(_zkClient, clusterName, instanceName, InstanceType.PARTICIPANT)) {
    throw new HelixException(
        "cluster " + clusterName + " instance " + instanceName + " is not setup yet");
  }
  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();

  InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceName));
  config.setZoneId(zoneId);
  accessor.setProperty(keyBuilder.instanceConfig(instanceName), config);
}
 

@Test(dependsOnMethods = "testGetClusters")
public void testGetClusterTopology() {
  System.out.println("Start test :" + TestHelper.getTestMethodName());
  String cluster = "TestCluster_1";
  String instance = cluster + "localhost_12920";
  // set the fake zone id in instance configuration
  HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(cluster, _baseAccessor);
  InstanceConfig instanceConfig =
      helixDataAccessor.getProperty(helixDataAccessor.keyBuilder().instanceConfig(instance));
  instanceConfig.setDomain("helixZoneId=123");
  helixDataAccessor.setProperty(helixDataAccessor.keyBuilder().instanceConfig(instance),
      instanceConfig);

  String response = new JerseyUriRequestBuilder("clusters/{}/topology").format(cluster).get(this);

  Assert.assertEquals(response,
      "{\"id\":\"TestCluster_1\",\"zones\":[{\"id\":\"123\",\"instances\":[{\"id\":\"TestCluster_1localhost_12920\"}]}],"
          + "\"allInstances\":[\"TestCluster_1localhost_12918\",\"TestCluster_1localhost_12919\",\"TestCluster_1localhost_12924\","
          + "\"TestCluster_1localhost_12925\",\"TestCluster_1localhost_12926\",\"TestCluster_1localhost_12927\",\"TestCluster_1localhost_12920\","
          + "\"TestCluster_1localhost_12921\",\"TestCluster_1localhost_12922\",\"TestCluster_1localhost_12923\"]}");
  System.out.println("End test :" + TestHelper.getTestMethodName());
}
 

/**
 * Reads HISTORY ZNode from the metadata store and generates a Map object that contains the
 * pertinent history entries depending on the history type.
 * @param clusterId
 * @param historyType
 * @return
 */
private Map<String, Object> getControllerHistory(String clusterId,
    ControllerHistory.HistoryType historyType) {
  HelixDataAccessor dataAccessor = getDataAccssor(clusterId);
  Map<String, Object> history = new HashMap<>();
  history.put(Properties.id.name(), clusterId);

  ControllerHistory historyRecord =
      dataAccessor.getProperty(dataAccessor.keyBuilder().controllerLeaderHistory());

  switch (historyType) {
    case CONTROLLER_LEADERSHIP:
      history.put(Properties.history.name(),
          historyRecord != null ? historyRecord.getHistoryList() : Collections.emptyList());
      break;
    case MAINTENANCE:
      history.put(ClusterProperties.maintenanceHistory.name(),
          historyRecord != null ? historyRecord.getMaintenanceHistoryList()
              : Collections.emptyList());
      break;
  }
  return history;
}
 

@Test public void testAddedFieldsInCurrentState() {
  String instanceName = PARTICIPANT_PREFIX + "_" + _startPort;
  HelixDataAccessor accessor = _manager.getHelixDataAccessor();
  LiveInstance liveInstance =
      accessor.getProperty(accessor.keyBuilder().liveInstance(instanceName));
  CurrentState currentState = accessor.getProperty(accessor.keyBuilder()
      .currentState(instanceName, liveInstance.getEphemeralOwner(), WorkflowGenerator.DEFAULT_TGT_DB));
  // Test start time should happen after test start time
  Assert.assertTrue(
      currentState.getStartTime(WorkflowGenerator.DEFAULT_TGT_DB + "_0") >= _testStartTime);

  // Test end time is always larger than start time
  Assert.assertTrue(
      currentState.getEndTime(WorkflowGenerator.DEFAULT_TGT_DB + "_0") >= currentState
          .getStartTime(WorkflowGenerator.DEFAULT_TGT_DB + "_0"));

  // Final state is MASTER, so SLAVE will be the previous state
  Assert.assertEquals(currentState.getPreviousState(WorkflowGenerator.DEFAULT_TGT_DB + "_0"),
      "SLAVE");
}
 

@GET
@Path("{clusterId}/controller/maintenanceSignal")
public Response getClusterMaintenanceSignal(@PathParam("clusterId") String clusterId) {
  HelixDataAccessor dataAccessor = getDataAccssor(clusterId);
  MaintenanceSignal maintenanceSignal =
      dataAccessor.getProperty(dataAccessor.keyBuilder().maintenance());
  if (maintenanceSignal != null) {
    Map<String, String> maintenanceInfo = maintenanceSignal.getRecord().getSimpleFields();
    maintenanceInfo.put(ClusterProperties.clusterName.name(), clusterId);
    return JSONRepresentation(maintenanceInfo);
  }
  return notFound(String.format("Cluster %s is not in maintenance mode!", clusterId));
}
 

StringRepresentation getClusterRepresentation(String clusterName) throws JsonGenerationException,
    JsonMappingException, IOException {
  ZkClient zkClient =
      ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.ZKCLIENT);
  ClusterSetup setupTool = new ClusterSetup(zkClient);
  List<String> instances =
      setupTool.getClusterManagementTool().getInstancesInCluster(clusterName);

  ZNRecord clusterSummayRecord = new ZNRecord("Cluster Summary");
  clusterSummayRecord.setListField("participants", instances);

  List<String> resources =
      setupTool.getClusterManagementTool().getResourcesInCluster(clusterName);
  clusterSummayRecord.setListField("resources", resources);

  List<String> models = setupTool.getClusterManagementTool().getStateModelDefs(clusterName);
  clusterSummayRecord.setListField("stateModelDefs", models);

  HelixDataAccessor accessor =
      ClusterRepresentationUtil.getClusterDataAccessor(zkClient, clusterName);
  Builder keyBuilder = accessor.keyBuilder();

  LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
  if (leader != null) {
    clusterSummayRecord.setSimpleField("LEADER", leader.getInstanceName());
  } else {
    clusterSummayRecord.setSimpleField("LEADER", "");
  }
  StringRepresentation representation =
      new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(clusterSummayRecord),
          MediaType.APPLICATION_JSON);

  return representation;
}
 

@GET
@Path("messages/{messageId}")
public Response getMessageOnInstance(@PathParam("clusterId") String clusterId,
    @PathParam("instanceName") String instanceName,
    @PathParam("messageId") String messageId) throws IOException {
  HelixDataAccessor accessor = getDataAccssor(clusterId);
  Message message = accessor.getProperty(accessor.keyBuilder().message(instanceName, messageId));
  if (message != null) {
    return JSONRepresentation(message.getRecord());
  }

  return notFound();
}
 

@Override
public boolean verify() {
  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(_clusterName, new ZkBaseDataAccessor<>(_gZkClient));
  Builder keyBuilder = accessor.keyBuilder();
  int numberOfPartitions;
  try {
    numberOfPartitions = accessor.getProperty(keyBuilder.idealStates(_resourceName)).getRecord()
        .getListFields().size();
  } catch (Exception e) {
    return false;
  }
  ResourceControllerDataProvider cache = new ResourceControllerDataProvider();
  cache.refresh(accessor);

  IdealState idealState = cache.getIdealState(_resourceName);
  if (idealState == null) {
    return false;
  }
  String masterValue =
      cache.getStateModelDef(idealState.getStateModelDefRef()).getStatesPriorityList().get(0);
  int replicas = Integer.parseInt(cache.getIdealState(_resourceName).getReplicas());
  String instanceGroupTag = cache.getIdealState(_resourceName).getInstanceGroupTag();
  int instances = 0;
  for (String liveInstanceName : cache.getLiveInstances().keySet()) {
    if (cache.getInstanceConfigMap().get(liveInstanceName).containsTag(instanceGroupTag)) {
      instances++;
    }
  }
  if (instances == 0) {
    instances = cache.getLiveInstances().size();
  }
  ExternalView ev = accessor.getProperty(keyBuilder.externalView(_resourceName));
  if (ev == null) {
    return false;
  }
  return verifyBalanceExternalView(ev.getRecord(), numberOfPartitions, masterValue, replicas,
      instances);
}
 

@GET
@Path("configs")
public Response getInstanceConfig(@PathParam("clusterId") String clusterId,
    @PathParam("instanceName") String instanceName) throws IOException {
  HelixDataAccessor accessor = getDataAccssor(clusterId);
  InstanceConfig instanceConfig =
      accessor.getProperty(accessor.keyBuilder().instanceConfig(instanceName));

  if (instanceConfig != null) {
    return JSONRepresentation(instanceConfig.getRecord());
  }

  return notFound();
}
 

@GET
@Path("healthreports/{reportName}")
public Response getHealthReportsOnInstance(
    @PathParam("clusterId") String clusterId, @PathParam("instanceName") String instanceName,
    @PathParam("reportName") String reportName) throws IOException {
  HelixDataAccessor accessor = getDataAccssor(clusterId);
  HealthStat healthStat =
      accessor.getProperty(accessor.keyBuilder().healthReport(instanceName, reportName));
  if (healthStat != null) {
    return JSONRepresentation(healthStat);
  }

  return notFound();
}
 

private void updateMaintenanceInfo(final HelixDataAccessor accessor) {
  _maintenanceSignal = accessor.getProperty(accessor.keyBuilder().maintenance());
  _isMaintenanceModeEnabled = _maintenanceSignal != null;
  // The following flag is to guarantee that there's only one update per pineline run because we
  // check for whether maintenance recovery could happen twice every pipeline
  _hasMaintenanceSignalChanged = false;
}
 

@Test()
public void testAutoRebalance() throws Exception {
  // kill 1 node
  _participants[0].syncStop();

  boolean result = ClusterStateVerifier
      .verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, TEST_DB));
  Assert.assertTrue(result);

  // add 2 nodes
  for (int i = 0; i < 2; i++) {
    String storageNodeName = PARTICIPANT_PREFIX + "_" + (1000 + i);
    _gSetupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);

    MockParticipantManager participant =
        new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, storageNodeName.replace(':', '_'));
    _extraParticipants.add(participant);
    participant.syncStart();
  }
  Thread.sleep(100);
  result = ClusterStateVerifier.verifyByPolling(
      new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, TEST_DB), 10000, 100);
  Assert.assertTrue(result);

  result = ClusterStateVerifier
      .verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, db2));
  Assert.assertTrue(result);
  HelixDataAccessor accessor =
      new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<>(_gZkClient));
  Builder keyBuilder = accessor.keyBuilder();
  ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2));
  Set<String> instancesSet = new HashSet<>();
  for (String partitionName : ev.getRecord().getMapFields().keySet()) {
    Map<String, String> assignmentMap = ev.getRecord().getMapField(partitionName);
    instancesSet.addAll(assignmentMap.keySet());
  }
  Assert.assertEquals(instancesSet.size(), 2);
}
 

/**
 * Get the customized state for a specified resource
 */
public CustomizedState getCustomizedState(String customizedStateName, String resourceName) {
  HelixDataAccessor accessor = _helixManager.getHelixDataAccessor();
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  return (CustomizedState) accessor
      .getProperty(keyBuilder.customizedState(_instanceName, customizedStateName, resourceName));
}
 

/**
 * Poll for the existence (or lack thereof) of a specific Helix property
 * @param clazz the HelixProeprty subclass
 * @param accessor connected HelixDataAccessor
 * @param key the property key to look up
 * @param shouldExist true if the property should exist, false otherwise
 * @return the property if found, or null if it does not exist
 */
protected <T extends HelixProperty> T pollForProperty(Class<T> clazz, HelixDataAccessor accessor,
    PropertyKey key, boolean shouldExist) throws InterruptedException {
  final int POLL_TIMEOUT = 5000;
  final int POLL_INTERVAL = 50;
  T property = accessor.getProperty(key);
  int timeWaited = 0;
  while (((shouldExist && property == null) || (!shouldExist && property != null))
      && timeWaited < POLL_TIMEOUT) {
    Thread.sleep(POLL_INTERVAL);
    timeWaited += POLL_INTERVAL;
    property = accessor.getProperty(key);
  }
  return property;
}
 

/**
 * Test addResourceWithWeight() and validateResourcesForWagedRebalance() by trying to add a resource with incomplete ResourceConfig.
 */
@Test
public void testAddResourceWithWeightAndValidation()
    throws IOException {
  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;
  String mockInstance = "MockInstance";
  String testResourcePrefix = "TestResource";
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  admin.addCluster(clusterName, true);
  admin.addStateModelDef(clusterName, "MasterSlave", new MasterSlaveSMD());

  // Create a dummy instance
  InstanceConfig instanceConfig = new InstanceConfig(mockInstance);
  Map<String, Integer> mockInstanceCapacity =
      ImmutableMap.of("WCU", 100, "RCU", 100, "STORAGE", 100);
  instanceConfig.setInstanceCapacityMap(mockInstanceCapacity);
  admin.addInstance(clusterName, instanceConfig);
  MockParticipantManager mockParticipantManager =
      new MockParticipantManager(ZK_ADDR, clusterName, mockInstance);
  mockParticipantManager.syncStart();

  IdealState idealState = new IdealState(testResourcePrefix);
  idealState.setNumPartitions(3);
  idealState.setStateModelDefRef("MasterSlave");
  idealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);

  ResourceConfig resourceConfig = new ResourceConfig(testResourcePrefix);
  // validate
  Map<String, Boolean> validationResult = admin.validateResourcesForWagedRebalance(clusterName,
      Collections.singletonList(testResourcePrefix));
  Assert.assertEquals(validationResult.size(), 1);
  Assert.assertFalse(validationResult.get(testResourcePrefix));
  try {
    admin.addResourceWithWeight(clusterName, idealState, resourceConfig);
    Assert.fail();
  } catch (HelixException e) {
    // OK since resourceConfig is empty
  }

  // Set PARTITION_CAPACITY_MAP
  Map<String, String> capacityDataMap =
      ImmutableMap.of("WCU", "1", "RCU", "2", "STORAGE", "3");
  resourceConfig.getRecord()
      .setMapField(ResourceConfig.ResourceConfigProperty.PARTITION_CAPACITY_MAP.name(),
          Collections.singletonMap(ResourceConfig.DEFAULT_PARTITION_KEY,
              OBJECT_MAPPER.writeValueAsString(capacityDataMap)));

  // validate
  validationResult = admin.validateResourcesForWagedRebalance(clusterName,
      Collections.singletonList(testResourcePrefix));
  Assert.assertEquals(validationResult.size(), 1);
  Assert.assertFalse(validationResult.get(testResourcePrefix));

  // Add the capacity key to ClusterConfig
  HelixDataAccessor dataAccessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
  PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
  ClusterConfig clusterConfig = dataAccessor.getProperty(keyBuilder.clusterConfig());
  clusterConfig.setInstanceCapacityKeys(Arrays.asList("WCU", "RCU", "STORAGE"));
  dataAccessor.setProperty(keyBuilder.clusterConfig(), clusterConfig);

  // Should succeed now
  Assert.assertTrue(admin.addResourceWithWeight(clusterName, idealState, resourceConfig));
  // validate
  validationResult = admin.validateResourcesForWagedRebalance(clusterName,
      Collections.singletonList(testResourcePrefix));
  Assert.assertEquals(validationResult.size(), 1);
  Assert.assertTrue(validationResult.get(testResourcePrefix));
}
 

@Test
public void testDataChange() throws Exception {
  // Create an admin and add LiveInstanceChange listener to it
  HelixManager adminManager =
      HelixManagerFactory.getZKHelixManager(clusterName, null, InstanceType.ADMINISTRATOR,
          ZK_ADDR);
  adminManager.connect();
  final BlockingQueue<List<LiveInstance>> changeList =
      new LinkedBlockingQueue<List<LiveInstance>>();

  adminManager.addLiveInstanceChangeListener(new LiveInstanceChangeListener() {
    @Override
    public void onLiveInstanceChange(List<LiveInstance> liveInstances,
        NotificationContext changeContext) {
      // The queue is basically unbounded, so shouldn't throw exception when calling
      // "add".
      changeList.add(deepCopy(liveInstances));
    }
  });

  // Check the initial condition
  List<LiveInstance> instances = changeList.poll(1, TimeUnit.SECONDS);
  Assert.assertNotNull(instances, "Expecting a list of live instance");
  Assert.assertTrue(instances.isEmpty(), "Expecting an empty list of live instance");
  // Join as participant, should trigger a live instance change event
  HelixManager manager =
      HelixManagerFactory.getZKHelixManager(clusterName, "localhost_54321",
          InstanceType.PARTICIPANT, ZK_ADDR);
  manager.connect();
  instances = changeList.poll(1, TimeUnit.SECONDS);
  Assert.assertNotNull(instances, "Expecting a list of live instance");
  Assert.assertEquals(instances.size(), 1, "Expecting one live instance");
  Assert.assertEquals(instances.get(0).getInstanceName(), manager.getInstanceName());
  // Update data in the live instance node, should trigger another live instance change
  // event
  HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
  PropertyKey propertyKey =
      helixDataAccessor.keyBuilder().liveInstance(manager.getInstanceName());
  LiveInstance instance = helixDataAccessor.getProperty(propertyKey);

  Map<String, String> map = new TreeMap<String, String>();
  map.put("k1", "v1");
  instance.getRecord().setMapField("test", map);
  Assert.assertTrue(helixDataAccessor.updateProperty(propertyKey, instance),
      "Failed to update live instance node");

  instances = changeList.poll(1, TimeUnit.SECONDS);
  Assert.assertNotNull(instances, "Expecting a list of live instance");
  Assert.assertEquals(instances.get(0).getRecord().getMapField("test"), map, "Wrong map data.");
  manager.disconnect();
  Thread.sleep(1000); // wait for callback finish

  instances = changeList.poll(1, TimeUnit.SECONDS);
  Assert.assertNotNull(instances, "Expecting a list of live instance");
  Assert.assertTrue(instances.isEmpty(), "Expecting an empty list of live instance");

  adminManager.disconnect();

}