Java Code Examples for org.apache.kafka.clients.admin.NewTopic#configs()

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    Map<String, String> topicConfigs = new HashMap<>();
    topicConfigs.put("retention.ms", Long.toString(Long.MAX_VALUE));

    NewTopic ratings = new NewTopic(envProps.getProperty("rating.topic.name"),
                                    Integer.parseInt(envProps.getProperty("rating.topic.partitions")),
                                    Short.parseShort(envProps.getProperty("rating.topic.replication.factor")));
    ratings.configs(topicConfigs);
    topics.add(ratings);

    NewTopic counts = new NewTopic(envProps.getProperty("rating.count.topic.name"),
                                   Integer.parseInt(envProps.getProperty("rating.count.topic.partitions")),
                                   Short.parseShort(envProps.getProperty("rating.count.topic.replication.factor")));
    counts.configs(topicConfigs);
    topics.add(counts);


    client.createTopics(topics);
    client.close();
}
 

private void createTopic(AdminClient adminClient, String topicName, Integer numberOfPartitions, Short replicationFactor) throws InterruptedException, ExecutionException {
    if (!adminClient.listTopics().names().get().contains(topicName)) {
        logger.info("Creating topic {}", topicName);

        final Map<String, String> configs = replicationFactor < 3 ? Map.of(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1") : Map.of();

        final NewTopic newTopic = new NewTopic(topicName, numberOfPartitions, replicationFactor);
        newTopic.configs(configs);
        try {
            CreateTopicsResult topicsCreationResult = adminClient.createTopics(Collections.singleton(newTopic));
            topicsCreationResult.all().get();
        } catch (ExecutionException e) {
            //silent ignore if topic already exists
        }
    }
}
 

@Test
public void shouldSetTopicCleanupPolicyToCompact() throws InterruptedException,
                                                          ExecutionException {
  expect(adminClient.listTopics()).andReturn(getEmptyListTopicResult());

  // Verify that the new topic configuration being passed to the admin client is what we expect.
  NewTopic newTopic = new NewTopic(topicName1, 1, (short) 1);
  newTopic.configs(Collections.singletonMap("cleanup.policy", "compact"));
  expect(adminClient.createTopics(singleNewTopic(newTopic))).andReturn(getCreateTopicsResult());
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  kafkaTopicClient.createTopic(topicName1,
                               1,
                               (short) 1,
                               Collections.singletonMap("cleanup.policy", "compact"));
  verify(adminClient);
}
 

public void createNewTopic(TopicToAdd topicToAdd) throws Exception {
    final String topicName = topicToAdd.getTopicName();
    Logger.trace(String.format("Creating topic '%s' [partitions:%d, replication factor:%d, cleanup policy:%s]",
                               topicName,
                               topicToAdd.getPartitions(),
                               topicToAdd.getReplicationFactor(),
                               topicToAdd.getCleanupPolicy()));

    final NewTopic newTopic = new NewTopic(topicName,
                                           topicToAdd.getPartitions(),
                                           (short) topicToAdd.getReplicationFactor());
    newTopic.configs(topicConfigsMapFromTopicToAdd(topicToAdd));

    final CreateTopicsResult result = kafkaClientsAdminClient.createTopics(Collections.singletonList(newTopic));
    interpretCreateTopicResult(topicName, result);
}
 

/**
 * Build a wrapper around the topic with the given desired properties and {@link #DEFAULT_CLEANUP_POLICY}.
 *
 * @param topic The name of the topic.
 * @param partitionCount Desired partition count.
 * @param replicationFactor Desired replication factor.
 * @param retentionMs Desired retention in milliseconds.
 * @return A wrapper around the topic with the given desired properties.
 */
public static NewTopic wrapTopic(String topic, int partitionCount, short replicationFactor, long retentionMs) {
  if (partitionCount <= 0 || replicationFactor <= 0 || retentionMs <= 0) {
    throw new IllegalArgumentException(String.format("Partition count (%d), replication factor (%d), and retention ms (%d)"
                                                     + " must be positive for the topic (%s).", partitionCount,
                                                     replicationFactor, retentionMs, topic));
  }

  NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
  Map<String, String> config = new HashMap<>(2);
  config.put(RetentionMsProp(), Long.toString(retentionMs));
  config.put(CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
  newTopic.configs(config);

  return newTopic;
}
 

protected NewTopic createMetricsTopicFromReporterConfig(CruiseControlMetricsReporterConfig reporterConfig)
    throws CruiseControlMetricsReporterException {
  String cruiseControlMetricsTopic =
      reporterConfig.getString(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG);
  Integer cruiseControlMetricsTopicNumPartition =
      reporterConfig.getInt(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_NUM_PARTITIONS_CONFIG);
  Short cruiseControlMetricsTopicReplicaFactor =
      reporterConfig.getShort(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_REPLICATION_FACTOR_CONFIG);
  if (cruiseControlMetricsTopicReplicaFactor <= 0 || cruiseControlMetricsTopicNumPartition <= 0) {
    throw new CruiseControlMetricsReporterException("The topic configuration must explicitly set the replication factor and the num partitions");
  }
  NewTopic newTopic = new NewTopic(cruiseControlMetricsTopic, cruiseControlMetricsTopicNumPartition, cruiseControlMetricsTopicReplicaFactor);

  Map<String, String> config = new HashMap<>(2);
  config.put(LogConfig.RetentionMsProp(),
             Long.toString(reporterConfig.getLong(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_RETENTION_MS_CONFIG)));
  config.put(LogConfig.CleanupPolicyProp(), CRUISE_CONTROL_METRICS_TOPIC_CLEAN_UP_POLICY);
  newTopic.configs(config);
  return newTopic;
}
 

/** Create a Kafka topic with the passed name. If compact is true then the cleanup policy is compact, delete.
 *  @param client {@link AdminClient}
 *  @param compact If the topic should be compacted.
 *  @param topic_name Name of the topic to be created.
 *  @return new_topic The newly created topic.
 */
private static NewTopic createTopic(final AdminClient client, final boolean compact, final String topic_name)
{
    final NewTopic new_topic = new NewTopic(topic_name, PARTITIONS, REPLICATION_FACTOR);
    final Map<String, String> configs = new HashMap<>();
    configs.put(segment_time, time);
    if (compact)
    {
        // Old talk messages are deleted.
        // Other messages are compacted and deleted when null
        if (topic_name.endsWith("Talk"))
            configs.put(cleanup_policy, delete_policy);
        else
            configs.put(cleanup_policy, compact_policy);
        configs.put(dirty2clean, ratio);
    }
    return new_topic.configs(configs);
}
 

private void createTopic(AdminClient admin) throws CacheInitializationException,
    InterruptedException, ExecutionException, TimeoutException {
    log.info("Creating topic {}", topic);

    int numLiveBrokers = admin.describeCluster().nodes().get(initTimeout, TimeUnit.MILLISECONDS).size();
    if (numLiveBrokers <= 0) {
        throw new CacheInitializationException("No live Kafka brokers");
    }

    int topicReplicationFactor = Math.min(numLiveBrokers, desiredReplicationFactor);
    if (topicReplicationFactor < desiredReplicationFactor) {
        log.warn("Creating the topic "
            + topic
            + " using a replication factor of "
            + topicReplicationFactor
            + ", which is less than the desired one of "
            + desiredReplicationFactor + ". If this is a production environment, it's "
            + "crucial to add more brokers and increase the replication factor of the topic.");
    }

    NewTopic topicRequest = new NewTopic(topic, desiredNumPartitions, (short) topicReplicationFactor);
    topicRequest.configs(
        Collections.singletonMap(
            TopicConfig.CLEANUP_POLICY_CONFIG,
            TopicConfig.CLEANUP_POLICY_COMPACT
        )
    );
    try {
        admin.createTopics(Collections.singleton(topicRequest)).all()
            .get(initTimeout, TimeUnit.MILLISECONDS);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof TopicExistsException) {
            // If topic already exists, ensure that it is configured correctly.
            verifyTopic(admin);
        } else {
            throw e;
        }
    }
}
 

@Bean
public NewTopic featureSetSpecsTopic(FeastProperties feastProperties) {
  StreamProperties streamProperties = feastProperties.getStream();
  Map<String, String> configs = new HashMap<>();
  configs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);

  NewTopic topic =
      new NewTopic(
          streamProperties.getSpecsOptions().getSpecsTopic(),
          DEFAULT_SPECS_TOPIC_PARTITIONING,
          DEFAULT_SPECS_TOPIC_REPLICATION);

  topic.configs(configs);
  return topic;
}
 

public boolean createTopic(TopicSpec spec, int timeOut) throws Exception {
  NewTopic newTopic = new NewTopic(
      spec.name(), spec.partitions(), (short) spec.replicationFactor()
  );
  newTopic.configs(spec.config());
  CreateTopicsResult result = adminClient.createTopics(
      Collections.singletonList(newTopic), new CreateTopicsOptions().timeoutMs(timeOut)
  );
  result.all().get();
  return true;
}
 

@SuppressWarnings("unchecked")
void maybeCreateTopic() throws Exception {
  if (_topicCreationEnabled) {
    int brokerCount = _adminClient.describeCluster().nodes().get().size();
    int numPartitions = Math.max((int) Math.ceil(brokerCount * _minPartitionsToBrokersRatio), minPartitionNum());
    NewTopic newTopic = new NewTopic(_topic, numPartitions, (short) _replicationFactor);
    newTopic.configs((Map) _topicProperties);
    _topicFactory.createTopicIfNotExist(_topic, (short) _replicationFactor, _minPartitionsToBrokersRatio,
        _topicProperties, _adminClient);
  }
}
 

/**
 * Create the topic. This method attempts to create a topic so that all
 * the brokers in the cluster will have partitionToBrokerRatio partitions.  If the topic exists, but has different parameters
 * then this does nothing to update the parameters.
 *
 * TODO: Do we care about rack aware mode?  I would think no because we want to spread the topic over all brokers.
 * @param topic topic name
 * @param replicationFactor the replication factor for the topic
 * @param partitionToBrokerRatio This is multiplied by the number brokers to compute the number of partitions in the topic.
 * @param minPartitionNum partition number to be created at least
 * @param topicConfig additional parameters for the topic for example min.insync.replicas
 * @param adminClient AdminClient object initialized.
 * @return the number of partitions created
 * @throws ExecutionException exception thrown then executing the topic creation fails.
 * @throws InterruptedException exception that's thrown when interrupt occurs.
 */
@SuppressWarnings("unchecked")
public static int createTopicIfNotExists(String topic, short replicationFactor, double partitionToBrokerRatio,
    int minPartitionNum, Properties topicConfig, AdminClient adminClient)
    throws ExecutionException, InterruptedException {
  try {
    if (adminClient.listTopics().names().get().contains(topic)) {
      LOG.info("AdminClient indicates that {} already exists in the cluster. Topic config: {}", topic, topicConfig);
      return getPartitionNumForTopic(adminClient, topic);
    }
    int brokerCount = Utils.getBrokerCount(adminClient);
    int partitionCount = Math.max((int) Math.ceil(brokerCount * partitionToBrokerRatio), minPartitionNum);
    try {
      NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
      //noinspection rawtypes
      newTopic.configs((Map) topicConfig);

      List<NewTopic> topics = new ArrayList<>();
      topics.add(newTopic);
      CreateTopicsResult result = adminClient.createTopics(topics);

      // waits for this topic creation future to complete, and then returns its result.
      result.values().get(topic).get();
      LOG.info("CreateTopicsResult: {}.", result.values());

    } catch (TopicExistsException e) {
      /* There is a race condition with the consumer. */
      LOG.info("Monitoring topic " + topic + " already exists in the cluster.", e);
      return getPartitionNumForTopic(adminClient, topic);
    }
    LOG.info("Created monitoring topic {} in cluster with {} partitions and replication factor of {}.", topic,
        partitionCount, replicationFactor);

    return partitionCount;
  } finally {
    LOG.info("Completed the topic creation if it doesn't exist for {}.", topic);
  }
}
 

@Override
public void createTopicIfNotExists(Topic topic, Map<String, String> config) {
    NewTopic newTopic = new NewTopic(topic.getName(), topic.getPartitions(), topic.getReplicationFactor());
    newTopic.configs(config);
    try {
        _adminClient.createTopics(Collections.singleton(newTopic)).all().get();
    } catch (ExecutionException | InterruptedException e) {
        if (e.getCause() instanceof TopicExistsException) {
            checkTopicPropertiesMatching(topic);
        } else {
            throw new RuntimeException(e);
        }
    }
}