Java Code Examples for org.apache.kafka.clients.admin.AdminClient#createTopics()

private void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();

  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();

  topics.add(new NewTopic(
      envProps.getProperty("input.avro.movies.topic.name"),
      parseInt(envProps.getProperty("input.avro.movies.topic.partitions")),
      parseShort(envProps.getProperty("input.avro.movies.topic.replication.factor"))));

  topics.add(new NewTopic(
      envProps.getProperty("output.proto.movies.topic.name"),
      parseInt(envProps.getProperty("output.proto.movies.topic.partitions")),
      parseShort(envProps.getProperty("output.proto.movies.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    Map<String, String> topicConfigs = new HashMap<>();
    topicConfigs.put("retention.ms", Long.toString(Long.MAX_VALUE));

    NewTopic ratings = new NewTopic(envProps.getProperty("rating.topic.name"),
                                    Integer.parseInt(envProps.getProperty("rating.topic.partitions")),
                                    Short.parseShort(envProps.getProperty("rating.topic.replication.factor")));
    ratings.configs(topicConfigs);
    topics.add(ratings);

    NewTopic counts = new NewTopic(envProps.getProperty("rating.count.topic.name"),
                                   Integer.parseInt(envProps.getProperty("rating.count.topic.partitions")),
                                   Short.parseShort(envProps.getProperty("rating.count.topic.replication.factor")));
    counts.configs(topicConfigs);
    topics.add(counts);


    client.createTopics(topics);
    client.close();
}
 

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 

public void createTopics(final Properties envProps) {
    final Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    final AdminClient client = AdminClient.create(config);

    final List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("album.topic.name"),
            Integer.parseInt(envProps.getProperty("album.topic.partitions")),
            Short.parseShort(envProps.getProperty("album.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("tracks.purchase.topic.name"),
            Integer.parseInt(envProps.getProperty("tracks.purchase.topic.partitions")),
            Short.parseShort(envProps.getProperty("tracks.purchase.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("music.interest.topic.name"),
            Integer.parseInt(envProps.getProperty("music.interest.topic.partitions")),
            Short.parseShort(envProps.getProperty("music.interest.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 

private void createTopic(AdminClient adminClient, String topicName, Integer numberOfPartitions, Short replicationFactor) throws InterruptedException, ExecutionException {
    if (!adminClient.listTopics().names().get().contains(topicName)) {
        logger.info("Creating topic {}", topicName);

        final Map<String, String> configs = replicationFactor < 3 ? Map.of(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1") : Map.of();

        final NewTopic newTopic = new NewTopic(topicName, numberOfPartitions, replicationFactor);
        newTopic.configs(configs);
        try {
            CreateTopicsResult topicsCreationResult = adminClient.createTopics(Collections.singleton(newTopic));
            topicsCreationResult.all().get();
        } catch (ExecutionException e) {
            //silent ignore if topic already exists
        }
    }
}
 

@Test
public void testCreateTopic() throws Exception {
        final String topic = "test";
        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
        properties.put("client.id", "test-consumer-id");
        properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        AdminClient client = KafkaAdminClient.create(properties);
        CreateTopicsResult result = client.createTopics(Arrays.asList(new NewTopic(topic, 1, (short) 1)));
        result.values().get(topic).get();
        for (Map.Entry<String, KafkaFuture<Void>> entry : result.values().entrySet()) {
            System.out.println("Create Topic : " + entry.getKey() + " " +
                    "isCancelled : " + entry.getValue().isCancelled() + " " +
                    "isCompletedExceptionally : " + entry.getValue().isCompletedExceptionally() + " " +
                    "isDone : " + entry.getValue().isDone());
        }
}
 

/** Create a topic for each of the topics in the passed list.
 *  @param client {@link AdminClient}
 *  @param compact If the topics should be compacted.
 *  @param topics_to_create {@link List} of {@link String}s filled with the names of topics to create.
 */
private static void createTopics(final AdminClient client, final boolean compact, final List<String> topics_to_create)
{
    // Create the new topics locally.
    final List<NewTopic> new_topics = new ArrayList<>();
    for (String topic : topics_to_create)
    {
            logger.info("Creating topic '" + topic + "'");
            new_topics.add(createTopic(client, compact, topic));
    }
    // Create the new topics in the Kafka server.
    try
    {
        final CreateTopicsResult res = client.createTopics(new_topics);
        final KafkaFuture<Void> future = res.all();
        future.get();
    }
    catch (Exception ex)
    {
        logger.log(Level.WARNING, "Attempt to create topics failed", ex);
    }
}
 

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.rock.topic.name"),
            Integer.parseInt(envProps.getProperty("input.rock.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.rock.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("input.classical.topic.name"),
            Integer.parseInt(envProps.getProperty("input.classical.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.classical.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 

private void createTopics(int targetPort, Integer numPartitions) throws InterruptedException, java.util.concurrent.ExecutionException {
    List<String> topics = embeddedConfiguration.getTopics();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Creating Kafka Topics in Embedded Kafka: {}", topics);
    }
    if (!topics.isEmpty()) {
        Properties properties = new Properties();
        properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ("127.0.0.1:" + targetPort));
        AdminClient adminClient = AdminClient.create(properties);
        final CreateTopicsResult result = adminClient.createTopics(topics.stream().map(s ->
                new NewTopic(s, numPartitions, (short) 1)).collect(Collectors.toList())
        );
        result.all().get();

        if (LOG.isInfoEnabled()) {
            LOG.info("Created Kafka Topics in Embedded Kafka: {}", topics);
        }
    }
}
 

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.drama.topic.name"),
            Integer.parseInt(envProps.getProperty("output.drama.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.drama.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.fantasy.topic.name"),
            Integer.parseInt(envProps.getProperty("output.fantasy.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.fantasy.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.other.topic.name"),
            Integer.parseInt(envProps.getProperty("output.other.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.other.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 

/**
 * Create the topic. This method attempts to create a topic so that all
 * the brokers in the cluster will have partitionToBrokerRatio partitions.  If the topic exists, but has different parameters
 * then this does nothing to update the parameters.
 *
 * TODO: Do we care about rack aware mode?  I would think no because we want to spread the topic over all brokers.
 * @param topic topic name
 * @param replicationFactor the replication factor for the topic
 * @param partitionToBrokerRatio This is multiplied by the number brokers to compute the number of partitions in the topic.
 * @param minPartitionNum partition number to be created at least
 * @param topicConfig additional parameters for the topic for example min.insync.replicas
 * @param adminClient AdminClient object initialized.
 * @return the number of partitions created
 * @throws ExecutionException exception thrown then executing the topic creation fails.
 * @throws InterruptedException exception that's thrown when interrupt occurs.
 */
@SuppressWarnings("unchecked")
public static int createTopicIfNotExists(String topic, short replicationFactor, double partitionToBrokerRatio,
    int minPartitionNum, Properties topicConfig, AdminClient adminClient)
    throws ExecutionException, InterruptedException {
  try {
    if (adminClient.listTopics().names().get().contains(topic)) {
      LOG.info("AdminClient indicates that {} already exists in the cluster. Topic config: {}", topic, topicConfig);
      return getPartitionNumForTopic(adminClient, topic);
    }
    int brokerCount = Utils.getBrokerCount(adminClient);
    int partitionCount = Math.max((int) Math.ceil(brokerCount * partitionToBrokerRatio), minPartitionNum);
    try {
      NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
      //noinspection rawtypes
      newTopic.configs((Map) topicConfig);

      List<NewTopic> topics = new ArrayList<>();
      topics.add(newTopic);
      CreateTopicsResult result = adminClient.createTopics(topics);

      // waits for this topic creation future to complete, and then returns its result.
      result.values().get(topic).get();
      LOG.info("CreateTopicsResult: {}.", result.values());

    } catch (TopicExistsException e) {
      /* There is a race condition with the consumer. */
      LOG.info("Monitoring topic " + topic + " already exists in the cluster.", e);
      return getPartitionNumForTopic(adminClient, topic);
    }
    LOG.info("Created monitoring topic {} in cluster with {} partitions and replication factor of {}.", topic,
        partitionCount, replicationFactor);

    return partitionCount;
  } finally {
    LOG.info("Completed the topic creation if it doesn't exist for {}.", topic);
  }
}
 

private void createTopic(AdminClient adminClient, String topicName, Integer numberOfPartitions, Short replicationFactor) throws InterruptedException, ExecutionException {
    if (!adminClient.listTopics().names().get().contains(topicName)) {
        logger.info("Creating topic {}", topicName);
        final NewTopic newTopic = new NewTopic(topicName, numberOfPartitions, replicationFactor);
        try {
            CreateTopicsResult topicsCreationResult = adminClient.createTopics(Collections.singleton(newTopic));
            topicsCreationResult.all().get();
        } catch (ExecutionException e) {
            //silent ignore if topic already exists
        }
    }
}
 

private void createTopics() {
  AdminClient adminClient = KafkaCruiseControlUtils.createAdminClient(Collections.singletonMap(
      AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, broker(0).plaintextAddr()));
  try {
    adminClient.createTopics(Arrays.asList(
        new NewTopic(TOPIC0, 2, (short) 2),
        new NewTopic(TOPIC1, 2, (short) 2)
    ));
  } finally {
    KafkaCruiseControlUtils.closeAdminClientWithTimeout(adminClient);
  }
}
 

/**
 * Creates the given topic if it does not exist.
 *
 * @param adminClient The adminClient to send createTopics request.
 * @param topicToBeCreated A wrapper around the topic to be created.
 * @return {@code false} if the topic to be created already exists, {@code true} otherwise.
 */
protected static boolean createTopic(AdminClient adminClient, NewTopic topicToBeCreated) {
  try {
    CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singletonList(topicToBeCreated));
    createTopicsResult.values().get(topicToBeCreated.name()).get(CruiseControlMetricsUtils.CLIENT_REQUEST_TIMEOUT_MS,
                                                                 TimeUnit.MILLISECONDS);
    LOG.info("Topic {} has been created.", topicToBeCreated.name());
  } catch (InterruptedException | ExecutionException | TimeoutException e) {
    if (e.getCause() instanceof TopicExistsException) {
      return false;
    }
    throw new IllegalStateException(String.format("Unable to create topic %s.", topicToBeCreated.name()), e);
  }
  return true;
}
 

private static void createKafkaTopicsInCluster(final AdminClient adminClient, final Properties envProps) {
  adminClient.createTopics(Arrays.asList(
          new NewTopic(
                  envProps.getProperty("input.topic.name"),
                  Integer.parseInt(envProps.getProperty("input.topic.partitions")),
                  Short.parseShort(envProps.getProperty("input.topic.replication.factor"))),
          new NewTopic(
                  envProps.getProperty("output.topic.name"),
                  Integer.parseInt(envProps.getProperty("output.topic.partitions")),
                  Short.parseShort(envProps.getProperty("output.topic.replication.factor")))));
}
 

public static void main(String[] args) {

        Config config = ConfigFactory.load();

        Properties properties = new Properties();

        properties.put("bootstrap.servers", config.getString("bootstrap.servers"));

        AdminClient client = AdminClient.create(properties);

        HashMap<String, NewTopic> topics = new HashMap<>();

        topics.put(
                config.getString("input.topic.name"),
                new NewTopic(
                        config.getString("input.topic.name"),
                        config.getNumber("input.topic.partitions").intValue(),
                        config.getNumber("input.topic.replication.factor").shortValue())
        );

        topics.put(
                config.getString("output.topic.name"),
                new NewTopic(
                        config.getString("output.topic.name"),
                        config.getNumber("output.topic.partitions").intValue(),
                        config.getNumber("output.topic.replication.factor").shortValue())
        );

        try {
            logger.info("Starting the topics creation");

            CreateTopicsResult result = client.createTopics(topics.values());

            result.values().forEach((topicName, future) -> {
                NewTopic topic = topics.get(topicName);
                future.whenComplete((aVoid, maybeError) ->
                        Optional
                                .ofNullable(maybeError)
                                .map(Try::<Void>failure)
                                .orElse(Try.successful(null))

                                .onFailure(throwable -> logger.error("Topic creation didn't complete:", throwable))
                                .onSuccess((anOtherVoid) -> logger.info(
                                        String.format(
                                                "Topic %s, has been successfully created " +
                                                        "with %s partitions and replicated %s times",
                                                topic.name(),
                                                topic.numPartitions(),
                                                topic.replicationFactor() - 1
                                        )
                                )));
            });

            result.all().get();
        } catch (InterruptedException | ExecutionException e) {
            if (!(e.getCause() instanceof TopicExistsException)) e.printStackTrace();
        } finally {
            client.close();
        }
    }
 

public static void createTopic(String topic, int numPartitions, short replicationFactor, Properties props) {
    NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
    AdminClient adminClient = AdminClient.create(props);
    adminClient.createTopics(Collections.singletonList(newTopic));
    adminClient.close();
}