kafka.server.ConfigType Java Examples

public KafkaTopicDetails topicDetails(String topic) {
  Map<Object, List<Object>> map = getPartitionInfo(topic);
  int numPartitions = map.size();
  int numReplicas = map.get(0).size();

  RoadType type;
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic);
  String cleanupPolicyConfig = topicConfig
      .getProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
  switch (cleanupPolicyConfig) {
  case TopicConfig.CLEANUP_POLICY_COMPACT:
    type = RoadType.COMPACT;
    break;
  default:
    type = RoadType.NORMAL;
  }
  log.debug("numPartitions: {}, numReplicas: {}", numPartitions, numReplicas);
  return new KafkaTopicDetails(type, numPartitions, numReplicas);
}
 

@Test
public void topic_updated_with_throttle_props() throws Exception {
  kafka.createTopic("test_topic4", 4, 1);
  JsonNode model = mapper
      .readTree(
          "{\"topicName\":\"test_topic4\",\"status\":{\"topicCreated\":true,\"partitions\":4,\"replicationFactor\":1}}");
  KafkaModelReader modelReader = context.getBean(KafkaModelReader.class);
  TrafficControl agent = context.getBean(TrafficControl.class);
  agent.inspectModel("test", modelReader.read(model));

  ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false);
  Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test_topic4");
  zkUtils.close();

  assertThat(config.getProperty("leader.replication.throttled.replicas"), is("*"));
  assertThat(config.getProperty("follower.replication.throttled.replicas"), is("*"));
}
 

@Test
public void topic_created_with_throttle_props() throws Exception {
  JsonNode model = mapper
      .readTree(
          "{\"topicName\":\"test_topic5\",\"status\":{\"topicCreated\":true,\"partitions\":4,\"replicationFactor\":1}}");
  KafkaModelReader modelReader = context.getBean(KafkaModelReader.class);
  TrafficControl agent = context.getBean(TrafficControl.class);
  agent.inspectModel("test", modelReader.read(model));

  ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false);
  Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test_topic5");
  zkUtils.close();

  assertThat(config.getProperty("leader.replication.throttled.replicas"), is("*"));
  assertThat(config.getProperty("follower.replication.throttled.replicas"), is("*"));
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
private static void verifyTopic(ZkUtils zkUtils, String topic) {
  Set topics = new HashSet();
  topics.add(topic);

  // check # partition and the replication factor
  scala.collection.mutable.Map partitionAssignmentForTopics = zkUtils
      .getPartitionAssignmentForTopics(JavaConversions.asScalaSet(topics).toSeq());
  scala.collection.Map partitionAssignment = (scala.collection.Map) partitionAssignmentForTopics.get(topic).get();

  if (partitionAssignment.size() != 1) {
    throw new RuntimeException(String.format("The schema topic %s should have only 1 partition.", topic));
  }

  // check the retention policy
  Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic);
  String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp());
  if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) {
    throw new RuntimeException(String.format("The retention policy of the schema topic %s must be compact.", topic));
  }
}
 

public static String getTopicProperty(final String topic, final String zkPath, final String propertyName) {
    try (KafkaZkClient zkClient = createZkClient(zkPath)) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        final Properties topicConfig = adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
        return topicConfig.getProperty(propertyName);
    }
}
 

public void checkAndUpdateTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), road.getTopicName());
  boolean modified = false;
  modified |= checkAndUpdateThrottledReplicas(topicConfig, LEADER_THROTTLED_REPLICAS);
  modified |= checkAndUpdateThrottledReplicas(topicConfig, FOLLOWER_THROTTLED_REPLICAS);
  if (modified) {
    AdminUtils.changeTopicConfig(zkUtils, road.getTopicName(), topicConfig);
    log.info("Updated topic {}", road.getTopicName());
  }
}
 

@Override
public void setRetentionTime(final String topic, final Long retentionMs) throws TopicConfigException {
    try (KafkaZkClient zkClient = createZkClient()) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        final Properties topicProps = adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
        topicProps.setProperty("retention.ms", Long.toString(retentionMs));
        adminZkClient.changeTopicConfig(topic, topicProps);
    } catch (final Exception e) {
        throw new TopicConfigException("Unable to update retention time for topic " + topic, e);
    }
}
 

/**
 * Consult Kafka to get the retention for a topic. This is not cached
 * in case the retention is changed externally after creation.
 * If no topic-level retention is configured, this method returns null.
 *
 * @param datastream Datastream
 * @return topic retention or null if no such config
 */
@Override
public Duration getRetention(Datastream datastream) {
  Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present");
  String destination = datastream.getDestination().getConnectionString();
  Validate.notNull(destination, "null destination URI");
  String topicName = KafkaTransportProviderUtils.getTopicName(destination);
  Properties props = AdminUtils.fetchEntityConfig(_zkUtils.get(), ConfigType.Topic(), topicName);
  if (!props.containsKey(TOPIC_RETENTION_MS)) {
    return null;
  }
  return Duration.ofMillis(Long.parseLong(props.getProperty(TOPIC_RETENTION_MS)));
}
 

private void removeThrottledRateFromBroker(Integer brokerId) {
  Properties config = _kafkaZkClient.getEntityConfigs(ConfigType.Broker(), String.valueOf(brokerId));
  Object oldLeaderThrottle = config.remove(LEADER_THROTTLED_RATE);
  Object oldFollowerThrottle = config.remove(FOLLOWER_THROTTLED_RATE);
  if (oldLeaderThrottle != null) {
    LOG.debug("Removing leader throttle on broker {}", brokerId);
  }
  if (oldFollowerThrottle != null) {
    LOG.debug("Removing follower throttle on broker {}", brokerId);
  }
  if (oldLeaderThrottle != null || oldFollowerThrottle != null) {
    ExecutorUtils.changeBrokerConfig(_adminZkClient, brokerId, config);
  }
}
 

private void removeAllThrottledReplicasFromTopic(String topic) {
  Properties config = _kafkaZkClient.getEntityConfigs(ConfigType.Topic(), topic);
  Object oldLeaderThrottle = config.remove(LEADER_THROTTLED_REPLICAS);
  Object oldFollowerThrottle = config.remove(FOLLOWER_THROTTLED_REPLICAS);
  if (oldLeaderThrottle != null) {
    LOG.debug("Removing leader throttled replicas for topic {}", topic);
  }
  if (oldFollowerThrottle != null) {
    LOG.debug("Removing follower throttled replicas for topic {}", topic);
  }
  if (oldLeaderThrottle != null || oldFollowerThrottle != null) {
    ExecutorUtils.changeTopicConfig(_adminZkClient, topic, config);
  }
}
 

private void setThrottledReplicas(String topic, Set<String> replicas, String configKey) {
  assert (configKey.equals(LEADER_THROTTLED_REPLICAS) || configKey.equals(FOLLOWER_THROTTLED_REPLICAS));
  Properties config = _kafkaZkClient.getEntityConfigs(ConfigType.Topic(), topic);
  // Merge new throttled replicas with existing configuration values.
  Set<String> newThrottledReplicas = new TreeSet<>(replicas);
  String oldThrottledReplicas = config.getProperty(configKey);
  if (oldThrottledReplicas != null) {
    newThrottledReplicas.addAll(Arrays.asList(oldThrottledReplicas.split(",")));
  }
  config.setProperty(configKey, String.join(",", newThrottledReplicas));
  ExecutorUtils.changeTopicConfig(_adminZkClient, topic, config);
}
 

private void setThrottledRateIfUnset(int brokerId, String configKey) {
  assert (_throttleRate != null);
  assert (configKey.equals(LEADER_THROTTLED_RATE) || configKey.equals(FOLLOWER_THROTTLED_RATE));
  Properties config = _kafkaZkClient.getEntityConfigs(ConfigType.Broker(), String.valueOf(brokerId));
  Object oldThrottleRate = config.setProperty(configKey, String.valueOf(_throttleRate));
  if (oldThrottleRate == null) {
    LOG.debug("Setting {} to {} bytes/second for broker {}", configKey, _throttleRate, brokerId);
    ExecutorUtils.changeBrokerConfig(_adminZkClient, brokerId, config);
  } else {
    LOG.debug("Not setting {} for broker {} because pre-existing throttle of {} was already set",
      configKey, brokerId, oldThrottleRate);
  }
}
 

@Override
public Properties topicConfigs(String topic) {
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(_connectString,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_GROUP,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_TYPE,
                                                                            _zkSecurityEnabled);
  try {
    AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
    return adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
  } finally {
    KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
  }
}
 

@Test
@SuppressWarnings("unchecked")
public void createTopic_withConfig() throws IOException {
    Properties topicConfig = new Properties();
    topicConfig.setProperty(LogConfig.CleanupPolicyProp(), "delete");

    client.createTopic(topic, 1, 1, topicConfig);

    assertThat(client.getPartitions(), hasItem(new TopicAndPartition(topic, 0)));

    Map<Object, Object> zkTopicConfig = OBJECT_MAPPER.readValue((String) KafkaTests.getZkUtils().zkClient()
            .readData(ZkUtils.getEntityConfigPath(ConfigType.Topic(), topic)), Map.class);
    assertThat(((Map<Object, Object>) zkTopicConfig.get("config")).get(LogConfig.CleanupPolicyProp()), is("delete"));
}
 

@Test
public void compact_road_created_correctly() throws Exception {
  KafkaRoad model = new KafkaRoad("test_topic6", "road.test_topic6", RoadType.COMPACT, null, null, false);

  context.getBean(TrafficControl.class).newModel("test_topic6", model);

  ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false);
  Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "road.test_topic6");
  zkUtils.close();

  assertThat(config.getProperty(TopicConfig.CLEANUP_POLICY_CONFIG), is(TopicConfig.CLEANUP_POLICY_COMPACT));
}
 

private void removeThrottledReplicasFromTopic(String topic, Set<String> replicas) {
  Properties config = _kafkaZkClient.getEntityConfigs(ConfigType.Topic(), topic);
  removeLeaderThrottledReplicasFromTopic(config, topic, replicas);
  removeFollowerThrottledReplicasFromTopic(config, topic, replicas);
  ExecutorUtils.changeTopicConfig(_adminZkClient, topic, config);
}
 

private void assertExpectedThrottledRateForBroker(KafkaZkClient kafkaZkClient, int broker, Long expectedRate) {
  Properties brokerConfig = kafkaZkClient.getEntityConfigs(ConfigType.Broker(), String.valueOf(broker));
  String expectedString = expectedRate == null ? null : String.valueOf(expectedRate);
  assertEquals(expectedString, brokerConfig.getProperty(ReplicationThrottleHelper.LEADER_THROTTLED_RATE));
  assertEquals(expectedString, brokerConfig.getProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_RATE));
}
 

private void assertExpectedThrottledReplicas(KafkaZkClient kafkaZkClient, String topic, String expectedReplicas) {
  Properties topicConfig = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), topic);
  assertEquals(expectedReplicas, topicConfig.getProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS));
  assertEquals(expectedReplicas, topicConfig.getProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS));
}
 

@Test
public void testAddingThrottlesWithPreExistingThrottles() throws InterruptedException {
  createTopics();
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(),
      "ReplicationThrottleHelperTestMetricGroup",
      "AddingThrottlesWithNoPreExistingThrottles",
          false);

  final long throttleRate = 100L;

  ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
  ExecutionProposal proposal = new ExecutionProposal(
      new TopicPartition(TOPIC0, 0),
      100,
      new ReplicaPlacementInfo(0),
      Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)),
      Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));

  ExecutionTask task = completedTaskForProposal(0, proposal);

  // Broker 0 has an existing leader and follower throttle; we expect these to be preserved.
  Properties broker0Config = new Properties();
  long preExistingBroker0ThrottleRate = 200L;
  broker0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
  broker0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
  ExecutorUtils.changeBrokerConfig(new AdminZkClient(kafkaZkClient), 0, broker0Config);

  // Partition 1 (which is not involved in any execution proposal) has pre-existing throttled
  // replicas (on both leaders and followers); we expect these configurations to be merged
  // with our new throttled replicas.
  Properties topic0Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC0);
  topic0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:0,1:1");
  topic0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:0,1:1");
  ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC0, topic0Config);

  // Topic 1 is not involved in any execution proposal. It has pre-existing throttled replicas.
  Properties topic1Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC1);
  topic1Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:1");
  topic1Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:1");
  ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC1, topic1Config);

  throttleHelper.setThrottles(Collections.singletonList(proposal));

  assertExpectedThrottledRateForBroker(kafkaZkClient, 0, preExistingBroker0ThrottleRate);
  assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
  assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
  // No throttle on broker 3 because it's not involved in any of the execution proposals:
  assertExpectedThrottledRateForBroker(kafkaZkClient, 3, null);
  // Existing throttled replicas are merged with new throttled replicas for topic 0:
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "0:0,0:1,0:2,1:0,1:1");
  // Existing throttled replicas are unchanged for topic 1:
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");

  throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());

  // We expect all throttles related to replica movement to be removed. Specifically,
  // any throttles related to partitions which were not moved will remain.
  // However, we do expect the broker throttles to be removed.
  throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
  Arrays.asList(0, 1, 2, 3).forEach((i) -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "1:0,1:1");
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");
}