Java Code Examples for org.apache.helix.model.InstanceConfig#getRecord()

/**
 * Check that updateInstanceConfig fails when there is no pre-existing InstanceConfig ZNode. This
 * is because InstanceConfig should have been created when the instance was added, and this REST
 * endpoint is not meant for creation.
 */
@Test(dependsOnMethods = "deleteInstanceConfig")
public void checkUpdateFails() throws IOException {
  System.out.println("Start test :" + TestHelper.getTestMethodName());
  String instanceName = CLUSTER_NAME + "non_existent_instance";
  InstanceConfig instanceConfig = new InstanceConfig(INSTANCE_NAME + "TEST");
  ZNRecord record = instanceConfig.getRecord();
  record.getSimpleFields().put("TestSimple", "value");
  record.getMapFields().put("TestMap", ImmutableMap.of("key", "value"));
  record.getListFields().put("TestList", Arrays.asList("e1", "e2", "e3"));

  Entity entity =
      Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE);
  new JerseyUriRequestBuilder("clusters/{}/instances/{}/configs")
      .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode())
      .format(CLUSTER_NAME, instanceName).post(this, entity);
  System.out.println("End test :" + TestHelper.getTestMethodName());
}
 

public static void main(String[] args) throws Exception {
  InstanceConfig localInstanceConfig = new InstanceConfig("localhost_12001");
  ZNRecord record = localInstanceConfig.getRecord();
  record.setSimpleField("change_log_dir", "data/localhost_12001/translog");
  record.setSimpleField("file_store_dir", "data/localhost_12001/filestore");
  record.setSimpleField("check_point_dir", "data/localhost_12001/checkpoint");
  InstanceConfig masterInstanceConfig = new InstanceConfig("localhost_12001");
  record = masterInstanceConfig.getRecord();
  record.setSimpleField("change_log_dir", "data/localhost_12000/translog");
  record.setSimpleField("file_store_dir", "data/localhost_12000/filestore");
  record.setSimpleField("check_point_dir", "data/localhost_12000/checkpoint");
  Replicator replicator = new Replicator(localInstanceConfig, "resource", "partition");
  replicator.startReplication(masterInstanceConfig);
}
 

/**
 * Test that when AdminOperation is specified to UpdateIdealState, Helix bootstrap tool updates IdealState only without
 * changing InstanceConfig.
 */
@Test
public void testUpdateIdealStateAdminOp() throws Exception {
  String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
  // Test regular bootstrap. This is to ensure InstanceConfig and IdealState are there before testing changing
  // IdealState (to trigger replica movement)
  long expectedResourceCount =
      (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
  writeBootstrapOrUpgrade(expectedResourceCount, false);

  // Now, change the replica count for two partitions.
  int totalPartitionCount = testPartitionLayout.getPartitionCount();
  int firstPartitionIndex = RANDOM.nextInt(totalPartitionCount);
  int secondPartitionIndex = (firstPartitionIndex + 1) % totalPartitionCount;
  List<PartitionId> allPartitions = testPartitionLayout.getPartitionLayout().getPartitions(null);
  Partition partition1 = (Partition) allPartitions.get(firstPartitionIndex);
  Partition partition2 = (Partition) allPartitions.get(secondPartitionIndex);

  // Add a new replica for partition1. Find a disk on a data node that does not already have a replica for partition1.
  HashSet<DataNodeId> partition1Nodes = new HashSet<>();
  for (ReplicaId replica : partition1.getReplicas()) {
    partition1Nodes.add(replica.getDataNodeId());
  }
  Disk diskForNewReplica;
  do {
    diskForNewReplica = testHardwareLayout.getRandomDisk();
  } while (partition1Nodes.contains(diskForNewReplica.getDataNode()) || !diskForNewReplica.getDataNode()
      .getDatacenterName()
      .equals("DC1"));
  // Add new replica into partition1
  ReplicaId replicaToAdd = new Replica(partition1, diskForNewReplica, testHardwareLayout.clusterMapConfig);
  partition1.addReplica(replicaToAdd);
  // Remove a replica from partition2.
  ReplicaId removedReplica = partition2.getReplicas().remove(0);

  String dcName = replicaToAdd.getDataNodeId().getDatacenterName();
  ZkInfo zkInfo = dcsToZkInfo.get(dcName);
  ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
  InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, getInstanceName(replicaToAdd.getDataNodeId()));
  // deep copy for subsequent verification
  InstanceConfig previousInstanceConfig = new InstanceConfig(instanceConfig.getRecord());
  Utils.writeJsonObjectToFile(zkJson, zkLayoutPath);
  Utils.writeJsonObjectToFile(testHardwareLayout.getHardwareLayout().toJSONObject(), hardwareLayoutPath);
  Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
  // upgrade Helix by updating IdealState: AdminOperation = UpdateIdealState
  HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath,
      CLUSTER_NAME_PREFIX, dcStr, DEFAULT_MAX_PARTITIONS_PER_RESOURCE, false, false, new HelixAdminFactory(), false,
      ClusterMapConfig.DEFAULT_STATE_MODEL_DEF, UpdateIdealState);
  verifyResourceCount(testHardwareLayout.getHardwareLayout(), expectedResourceCount);

  // verify IdealState has been updated
  // 1. new added replica is indeed present in the IdealState
  verifyIdealStateForPartition(replicaToAdd, true, 4, expectedResourceCount);
  // 2. removed old replica is no longer present in the IdealState
  verifyIdealStateForPartition(removedReplica, false, 2, expectedResourceCount);

  // verify the InstanceConfig stays unchanged
  InstanceConfig currentInstanceConfig =
      admin.getInstanceConfig(clusterName, getInstanceName(replicaToAdd.getDataNodeId()));
  assertEquals("InstanceConfig should stay unchanged", previousInstanceConfig.getRecord(),
      currentInstanceConfig.getRecord());
}
 

/**
 * Test when AdminOperation is specified to DisablePartition, Helix bootstrap tool is able to disable certain partition
 * only without changing IdealState and InstanceConfig. (In practice, this is first step to decommission a replica)
 * @throws Exception
 */
@Test
public void testDisablePartitionAdminOp() throws Exception {
  String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
  // Test regular bootstrap. This is to ensure InstanceConfig and IdealState are there before testing disabling certain
  // replica on specific node.
  long expectedResourceCount =
      (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
  writeBootstrapOrUpgrade(expectedResourceCount, false);
  int totalPartitionCount = testPartitionLayout.getPartitionCount();
  // Randomly pick a partition to remove one of its replicas
  Partition testPartition = (Partition) testPartitionLayout.getPartitionLayout()
      .getPartitions(null)
      .get(RANDOM.nextInt(totalPartitionCount));

  ReplicaId removedReplica = testPartition.getReplicaIds()
      .stream()
      .filter(r -> r.getDataNodeId().getDatacenterName().equals("DC1"))
      .findFirst()
      .get();
  testPartition.getReplicas().remove(removedReplica);
  ZkInfo zkInfo = dcsToZkInfo.get(removedReplica.getDataNodeId().getDatacenterName());
  ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
  InstanceConfig instanceConfig =
      admin.getInstanceConfig(clusterName, getInstanceName(removedReplica.getDataNodeId()));
  // Deep copy the InstanceConfig for validation
  InstanceConfig previousInstanceConfig = new InstanceConfig(instanceConfig.getRecord());
  // Write changes to static files
  Utils.writeJsonObjectToFile(zkJson, zkLayoutPath);
  Utils.writeJsonObjectToFile(testHardwareLayout.getHardwareLayout().toJSONObject(), hardwareLayoutPath);
  Utils.writeJsonObjectToFile(testPartitionLayout.getPartitionLayout().toJSONObject(), partitionLayoutPath);
  // Upgrade Helix by updating IdealState: AdminOperation = DisablePartition
  HelixBootstrapUpgradeUtil.bootstrapOrUpgrade(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath,
      CLUSTER_NAME_PREFIX, dcStr, DEFAULT_MAX_PARTITIONS_PER_RESOURCE, false, false, new HelixAdminFactory(), false,
      ClusterMapConfig.DEFAULT_STATE_MODEL_DEF, DisablePartition);
  verifyResourceCount(testHardwareLayout.getHardwareLayout(), expectedResourceCount);
  // Verify that IdealState has no change
  verifyIdealStateForPartition(removedReplica, true, 3, expectedResourceCount);
  // Verify the InstanceConfig is changed only in MapFields (Disabled partitions are added to this field)
  InstanceConfig currentInstanceConfig =
      admin.getInstanceConfig(clusterName, getInstanceName(removedReplica.getDataNodeId()));
  String disabledPartitionStr = currentInstanceConfig.getRecord()
      .getMapFields()
      .keySet()
      .stream()
      .filter(k -> !k.startsWith("/mnt"))
      .findFirst()
      .get();
  // Deep copy the current InstanceConfig to remove disabled partitions entry and compare it with previous InstanceConfig
  InstanceConfig currentCopy = new InstanceConfig(currentInstanceConfig.getRecord());
  currentCopy.getRecord().getMapFields().remove(disabledPartitionStr);
  assertEquals("InstanceConfig should stay unchanged after disabled partition entry is removed",
      previousInstanceConfig.getRecord(), currentCopy.getRecord());
  // Verify that replica has been disabled
  String resourceName = null;
  for (String rs : admin.getResourcesInCluster(clusterName)) {
    IdealState is = admin.getResourceIdealState(clusterName, rs);
    if (is.getPartitionSet().contains(removedReplica.getPartitionId().toPathString())) {
      resourceName = rs;
      break;
    }
  }
  List<String> disabledPartition = currentInstanceConfig.getDisabledPartitions(resourceName);
  assertEquals("Disabled partition is not expected",
      Collections.singletonList(removedReplica.getPartitionId().toPathString()), disabledPartition);
}