org.apache.kafka.clients.admin.NewTopic Java Exaples

Source File: TopologyBuilderAdminClient.java From kafka-topology-builder with MIT License

7 votes

public void createTopic(Topic topic, String fullTopicName) throws IOException {

    int numPartitions =
        Integer.parseInt(topic.getConfig().getOrDefault(TopicManager.NUM_PARTITIONS, "3"));
    short replicationFactor =
        Short.parseShort(topic.getConfig().getOrDefault(TopicManager.REPLICATION_FACTOR, "2"));

    NewTopic newTopic =
        new NewTopic(fullTopicName, numPartitions, replicationFactor).configs(topic.rawConfig());
    Collection<NewTopic> newTopics = Collections.singleton(newTopic);
    try {
      createAllTopics(newTopics);
    } catch (ExecutionException | InterruptedException e) {
      LOGGER.error(e);
      throw new IOException(e);
    }
  }

Source File: KafkaTestContainerManager.java From apicurio-registry with Apache License 2.0

7 votes

private void createTopics(String bootstrapServers) {
    Properties properties = new Properties();
    properties.put("bootstrap.servers", bootstrapServers);
    properties.put("connections.max.idle.ms", 10000);
    properties.put("request.timeout.ms", 5000);
    try (AdminClient client = AdminClient.create(properties)) {
        CreateTopicsResult result = client.createTopics(Arrays.asList(
                new NewTopic("storage-topic", 1, (short) 1),
                new NewTopic("global-id-topic", 1, (short) 1),
                new NewTopic("snapshot-topic", 1, (short) 1)
        ));
        try {
            result.all().get();
        } catch ( InterruptedException | ExecutionException e ) {
            throw new IllegalStateException(e);
        }
    }
}

Source File: NamingChangelogAndRepartitionTopics.java From kafka-tutorials with Apache License 2.0

7 votes

public void createTopics(final Properties envProps) {
  final Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  try (final AdminClient client = AdminClient.create(config)) {

    final List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
        envProps.getProperty("input.topic.name"),
        Integer.parseInt(envProps.getProperty("input.topic.partitions")),
        Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

    topics.add(new NewTopic(
        envProps.getProperty("output.topic.name"),
        Integer.parseInt(envProps.getProperty("output.topic.partitions")),
        Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    topics.add(new NewTopic(
        envProps.getProperty("join.topic.name"),
        Integer.parseInt(envProps.getProperty("join.topic.partitions")),
        Short.parseShort(envProps.getProperty("join.topic.replication.factor"))));

    client.createTopics(topics);
  }
}

Source File: CruiseControlMetricsReporter.java From cruise-control with BSD 2-Clause "Simplified" License

6 votes

protected NewTopic createMetricsTopicFromReporterConfig(CruiseControlMetricsReporterConfig reporterConfig)
    throws CruiseControlMetricsReporterException {
  String cruiseControlMetricsTopic =
      reporterConfig.getString(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG);
  Integer cruiseControlMetricsTopicNumPartition =
      reporterConfig.getInt(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_NUM_PARTITIONS_CONFIG);
  Short cruiseControlMetricsTopicReplicaFactor =
      reporterConfig.getShort(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_REPLICATION_FACTOR_CONFIG);
  if (cruiseControlMetricsTopicReplicaFactor <= 0 || cruiseControlMetricsTopicNumPartition <= 0) {
    throw new CruiseControlMetricsReporterException("The topic configuration must explicitly set the replication factor and the num partitions");
  }
  NewTopic newTopic = new NewTopic(cruiseControlMetricsTopic, cruiseControlMetricsTopicNumPartition, cruiseControlMetricsTopicReplicaFactor);

  Map<String, String> config = new HashMap<>(2);
  config.put(LogConfig.RetentionMsProp(),
             Long.toString(reporterConfig.getLong(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_RETENTION_MS_CONFIG)));
  config.put(LogConfig.CleanupPolicyProp(), CRUISE_CONTROL_METRICS_TOPIC_CLEAN_UP_POLICY);
  newTopic.configs(config);
  return newTopic;
}

Source File: RunningAverage.java From kafka-tutorials with Apache License 2.0

6 votes

/**
 * Create topics using AdminClient API
 */
private void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();

  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();

  topics.add(new NewTopic(
      envProps.getProperty("input.ratings.topic.name"),
      parseInt(envProps.getProperty("input.ratings.topic.partitions")),
      parseShort(envProps.getProperty("input.ratings.topic.replication.factor"))));

  topics.add(new NewTopic(
      envProps.getProperty("output.rating-averages.topic.name"),
      parseInt(envProps.getProperty("output.rating-averages.topic.partitions")),
      parseShort(envProps.getProperty("output.rating-averages.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();

}

Source File: FindDistinctEvents.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}

Source File: AdminClientWrapper.java From hdinsight-kafka-java-get-started with MIT License

6 votes

public static void createTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        int numPartitions = 8;
        short replicationFactor = (short)3;
        final NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);

        final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));
        createTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " created");
    } catch (Exception e) {
        System.out.print("Create Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}

Source File: KafkaChannelAutoConfiguration.java From servicecomb-pack with Apache License 2.0

6 votes

@PostConstruct
public void init() {
  Map props = new HashMap<>();
  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers);
  props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 50000);
  try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
    try {
      final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
      final CreateTopicsResult createTopicsResult = adminClient
          .createTopics(Collections.singleton(newTopic));
      createTopicsResult.values().get(topic).get();
    } catch (InterruptedException | ExecutionException e) {
      if (e.getCause() instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }
      if (!(e.getCause() instanceof TopicExistsException)) {
        throw new RuntimeException(e.getMessage(), e);
      }
    }
  }
  LOG.info("Kafka Channel Init");
}

Source File: TopicServiceImplTest.java From kafka-helmsman with MIT License

6 votes

@Test
public void testCreate() {
  TopicService service = new TopicServiceImpl(adminClient, true);
  CreateTopicsResult createTopicsResult = mock(CreateTopicsResult.class);
  when(createTopicsResult.all()).thenReturn(KafkaFuture.completedFuture(null));
  when(adminClient.createTopics(any(Collection.class),
      any(CreateTopicsOptions.class))).thenReturn(createTopicsResult);

  service.create(Collections.singletonList(
      new ConfiguredTopic("test", 1, (short) 2, Collections.emptyMap())));

  ArgumentCaptor<List> newTopics = ArgumentCaptor.forClass(List.class);
  ArgumentCaptor<CreateTopicsOptions> options = ArgumentCaptor.forClass(CreateTopicsOptions.class);
  verify(adminClient).createTopics((Collection<NewTopic>) newTopics.capture(), options.capture());
  Assert.assertEquals(1, newTopics.getValue().size());
  Assert.assertEquals("test", ((NewTopic) newTopics.getValue().get(0)).name());
  Assert.assertEquals(2, ((NewTopic) newTopics.getValue().get(0)).replicationFactor());
  Assert.assertTrue(options.getValue().shouldValidateOnly());
}

Source File: TopicSerializationTest.java From strimzi-kafka-operator with Apache License 2.0

6 votes

@Test
public void testToNewTopic() {
    Topic topic = new Topic.Builder()
            .withTopicName("test-topic")
            .withConfigEntry("foo", "bar")
            .withNumPartitions(3)
            .withNumReplicas((short) 2)
            .withMapName("gee")
            .build();
    NewTopic newTopic = TopicSerialization.toNewTopic(topic, null);
    assertThat(newTopic.name(), is("test-topic"));
    assertThat(newTopic.numPartitions(), is(3));
    assertThat(newTopic.replicationFactor(), is((short) 2));
    assertThat(newTopic.replicasAssignments(), is(nullValue()));
    assertThat(newTopic.configs(), is(singletonMap("foo", "bar")));
}

Source File: TransformEvents.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {

        Map<String, Object> config = new HashMap<>();
        config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));

        try (AdminClient adminClient = AdminClient.create(config)) {

            List<NewTopic> topics = new ArrayList<>();
            topics.add(new NewTopic(
                    envProps.getProperty("input.topic.name"),
                    Integer.parseInt(envProps.getProperty("input.topic.partitions")),
                    Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
            topics.add(new NewTopic(
                    envProps.getProperty("output.topic.name"),
                    Integer.parseInt(envProps.getProperty("output.topic.partitions")),
                    Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));
    
            adminClient.createTopics(topics);

        }

    }

Source File: StreamsIngest.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}

Source File: FkJoinTableToTable.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(final Properties envProps) {
    final Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    final AdminClient client = AdminClient.create(config);

    final List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("album.topic.name"),
            Integer.parseInt(envProps.getProperty("album.topic.partitions")),
            Short.parseShort(envProps.getProperty("album.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("tracks.purchase.topic.name"),
            Integer.parseInt(envProps.getProperty("tracks.purchase.topic.partitions")),
            Short.parseShort(envProps.getProperty("tracks.purchase.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("music.interest.topic.name"),
            Integer.parseInt(envProps.getProperty("music.interest.topic.partitions")),
            Short.parseShort(envProps.getProperty("music.interest.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}

Source File: AggregatingSum.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}

Source File: TumblingWindow.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    Map<String, String> topicConfigs = new HashMap<>();
    topicConfigs.put("retention.ms", Long.toString(Long.MAX_VALUE));

    NewTopic ratings = new NewTopic(envProps.getProperty("rating.topic.name"),
                                    Integer.parseInt(envProps.getProperty("rating.topic.partitions")),
                                    Short.parseShort(envProps.getProperty("rating.topic.replication.factor")));
    ratings.configs(topicConfigs);
    topics.add(ratings);

    NewTopic counts = new NewTopic(envProps.getProperty("rating.count.topic.name"),
                                   Integer.parseInt(envProps.getProperty("rating.count.topic.partitions")),
                                   Short.parseShort(envProps.getProperty("rating.count.topic.replication.factor")));
    counts.configs(topicConfigs);
    topics.add(counts);


    client.createTopics(topics);
    client.close();
}

Source File: AggregatingCount.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}

Source File: TransformStream.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}

Source File: MergeStreams.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.rock.topic.name"),
            Integer.parseInt(envProps.getProperty("input.rock.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.rock.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("input.classical.topic.name"),
            Integer.parseInt(envProps.getProperty("input.classical.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.classical.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}

Source File: SamplingUtilsTest.java From cruise-control with BSD 2-Clause "Simplified" License

6 votes

@Test
public void testMaybeIncreasePartitionCount() throws InterruptedException, ExecutionException, TimeoutException {
  AdminClient adminClient = EasyMock.createMock(AdminClient.class);
  NewTopic topicToAddPartitions = SamplingUtils.wrapTopic(MOCK_TOPIC, MOCK_DESIRED_PARTITION_COUNT,
                                                          MOCK_REPLICATION_FACTOR, MOCK_DESIRED_RETENTION_MS);
  DescribeTopicsResult describeTopicsResult = EasyMock.createMock(DescribeTopicsResult.class);
  KafkaFuture<TopicDescription> topicDescriptionFuture = EasyMock.createMock(KafkaFuture.class);
  TopicDescription topicDescription = EasyMock.createMock(TopicDescription.class);
  Map<String, KafkaFuture<TopicDescription>> describeTopicsValues = Collections.singletonMap(MOCK_TOPIC, topicDescriptionFuture);
  Map<String, KafkaFuture<Void>> createPartitionsValues = Collections.singletonMap(MOCK_TOPIC, EasyMock.createMock(KafkaFuture.class));
  CreatePartitionsResult createPartitionsResult = EasyMock.createMock(CreatePartitionsResult.class);

  EasyMock.expect(adminClient.describeTopics(Collections.singletonList(MOCK_TOPIC))).andReturn(describeTopicsResult);
  EasyMock.expect(describeTopicsResult.values()).andReturn(describeTopicsValues);
  EasyMock.expect(topicDescriptionFuture.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(topicDescription);
  EasyMock.expect(topicDescription.partitions()).andReturn(MOCK_PARTITIONS);
  EasyMock.expect(adminClient.createPartitions(Collections.singletonMap(MOCK_TOPIC, EasyMock.anyObject())))
          .andReturn(createPartitionsResult);
  EasyMock.expect(createPartitionsResult.values()).andReturn(createPartitionsValues);

  EasyMock.replay(adminClient, describeTopicsResult, topicDescriptionFuture, topicDescription, createPartitionsResult);
  boolean increasePartitionCount = SamplingUtils.maybeIncreasePartitionCount(adminClient, topicToAddPartitions);

  EasyMock.verify(adminClient, describeTopicsResult, topicDescriptionFuture, topicDescription, createPartitionsResult);
  assertTrue(increasePartitionCount);
}

Source File: DynamicOutputTopic.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(final Properties envProps) {
    final Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    try (final AdminClient client = AdminClient.create(config)) {

    final List<NewTopic> topics = new ArrayList<>();

        topics.add(new NewTopic(
                envProps.getProperty("input.topic.name"),
                Integer.parseInt(envProps.getProperty("input.topic.partitions")),
                Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

        topics.add(new NewTopic(
                envProps.getProperty("output.topic.name"),
                Integer.parseInt(envProps.getProperty("output.topic.partitions")),
                Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

        topics.add(new NewTopic(
            envProps.getProperty("special.order.topic.name"),
            Integer.parseInt(envProps.getProperty("special.order.topic.partitions")),
            Short.parseShort(envProps.getProperty("special.order.topic.replication.factor"))));

        client.createTopics(topics);
    }
}

Source File: FilterEvents.java From kafka-tutorials with Apache License 2.0

6 votes

public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}

Source File: IntegrationTest.java From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0

6 votes

@BeforeEach
void setUp() throws ExecutionException, InterruptedException {
    testBucketAccessor.clear(gcsPrefix);

    final Properties adminClientConfig = new Properties();
    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
    adminClient = AdminClient.create(adminClientConfig);

    final Map<String, Object> producerProps = new HashMap<>();
    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.ByteArraySerializer");
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.ByteArraySerializer");
    producer = new KafkaProducer<>(producerProps);

    final NewTopic newTopic0 = new NewTopic(TEST_TOPIC_0, 4, (short) 1);
    final NewTopic newTopic1 = new NewTopic(TEST_TOPIC_1, 4, (short) 1);
    adminClient.createTopics(Arrays.asList(newTopic0, newTopic1)).all().get();

    connectRunner = new ConnectRunner(pluginDir, kafka.getBootstrapServers(), OFFSET_FLUSH_INTERVAL_MS);
    connectRunner.start();
}

Source File: SamplingUtils.java From cruise-control with BSD 2-Clause "Simplified" License

6 votes

/**
 * Build a wrapper around the topic with the given desired properties and {@link #DEFAULT_CLEANUP_POLICY}.
 *
 * @param topic The name of the topic.
 * @param partitionCount Desired partition count.
 * @param replicationFactor Desired replication factor.
 * @param retentionMs Desired retention in milliseconds.
 * @return A wrapper around the topic with the given desired properties.
 */
public static NewTopic wrapTopic(String topic, int partitionCount, short replicationFactor, long retentionMs) {
  if (partitionCount <= 0 || replicationFactor <= 0 || retentionMs <= 0) {
    throw new IllegalArgumentException(String.format("Partition count (%d), replication factor (%d), and retention ms (%d)"
                                                     + " must be positive for the topic (%s).", partitionCount,
                                                     replicationFactor, retentionMs, topic));
  }

  NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
  Map<String, String> config = new HashMap<>(2);
  config.put(RetentionMsProp(), Long.toString(retentionMs));
  config.put(CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
  newTopic.configs(config);

  return newTopic;
}

Source File: KafkaEmbedded.java From micronaut-kafka with Apache License 2.0

6 votes

private void createTopics(int targetPort, Integer numPartitions) throws InterruptedException, java.util.concurrent.ExecutionException {
    List<String> topics = embeddedConfiguration.getTopics();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Creating Kafka Topics in Embedded Kafka: {}", topics);
    }
    if (!topics.isEmpty()) {
        Properties properties = new Properties();
        properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ("127.0.0.1:" + targetPort));
        AdminClient adminClient = AdminClient.create(properties);
        final CreateTopicsResult result = adminClient.createTopics(topics.stream().map(s ->
                new NewTopic(s, numPartitions, (short) 1)).collect(Collectors.toList())
        );
        result.all().get();

        if (LOG.isInfoEnabled()) {
            LOG.info("Created Kafka Topics in Embedded Kafka: {}", topics);
        }
    }
}

Source File: TopicAdmin.java From kafka-message-tool with MIT License

6 votes

public void createNewTopic(TopicToAdd topicToAdd) throws Exception {
    final String topicName = topicToAdd.getTopicName();
    Logger.trace(String.format("Creating topic '%s' [partitions:%d, replication factor:%d, cleanup policy:%s]",
                               topicName,
                               topicToAdd.getPartitions(),
                               topicToAdd.getReplicationFactor(),
                               topicToAdd.getCleanupPolicy()));

    final NewTopic newTopic = new NewTopic(topicName,
                                           topicToAdd.getPartitions(),
                                           (short) topicToAdd.getReplicationFactor());
    newTopic.configs(topicConfigsMapFromTopicToAdd(topicToAdd));

    final CreateTopicsResult result = kafkaClientsAdminClient.createTopics(Collections.singletonList(newTopic));
    interpretCreateTopicResult(topicName, result);
}

Source File: KafkaRule.java From kafka-pubsub-emulator with Apache License 2.0

6 votes

public void createTopic(String name) throws Exception {
  try (AdminClient admin = AdminClient.create(getAdminConfig())) {
    int partitions = 3;
    int replication = this.replicationFactor;
    Matcher matcher = Pattern.compile("(-\\d+[p|r])").matcher(name);
    while (matcher.find()) {
      String group = matcher.group();
      if (group.endsWith("p")) {
        partitions = getNumber(group);
      } else {
        replication = getNumber(group);
      }
    }

    NewTopic topic = new NewTopic(name, partitions, (short) replication);
    CreateTopicsResult topicsResult = admin.createTopics(singleton(topic));
    topicsResult.all().get(5, TimeUnit.SECONDS);
  } catch (Exception e) {
    LOGGER.log(Level.SEVERE, "Error on create topics " + name, e);
    throw e;
  }
}

Source File: MockKafka.java From strimzi-kafka-operator with Apache License 2.0

6 votes

@Override
public Future<Void> createTopic(Topic t) {
    NewTopic newTopic = TopicSerialization.toNewTopic(t, null);
    Future<Void> event = createTopicResponse.apply(newTopic.name());
    if (event.succeeded()) {
        Topic.Builder topicBuilder = new Topic.Builder()
                .withTopicName(newTopic.name())
                .withNumPartitions(newTopic.numPartitions())
                .withNumReplicas(newTopic.replicationFactor())
                .withMetadata(t.getMetadata());
        try {
            Field field = NewTopic.class.getDeclaredField("configs");
            field.setAccessible(true);
            topicBuilder.withConfig((Map) field.get(newTopic));
        } catch (ReflectiveOperationException e) {
            throw new RuntimeException(e);
        }
        Topic topic = topicBuilder.build();
        topics.put(topic.getTopicName(), topic);
    }
    return event;
}

Source File: NewTopicProperties.java From stream-registry with Apache License 2.0

6 votes

public NewTopic buildNewTopic(final String topicName) {
  // We execute this method and its validations only when 'notification.events.kafka.topic.setup' is true
  // during KafkaSetupHandler building. If we use @Validated and its constraints, bean loading will fail even
  // when topic setup is disabled.
  val component = "enabled Kafka topic setup";

  Objects.requireNonNull(topicName, getWarningMessageOnNotDefinedProp(component, KAFKA_TOPIC_NAME_PROPERTY));
  Objects.requireNonNull(numPartitions, getWarningMessageOnNotDefinedProp(component,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".numPartitions")));
  Objects.requireNonNull(replicationFactor, getWarningMessageOnNotDefinedProp(component,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".replicationFactor")));

  val gtZeroWarning = " must be greater than zero";

  Preconditions.checkArgument(numPartitions.compareTo(0) > 0,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".numPartitions").concat(gtZeroWarning));
  Preconditions.checkArgument(replicationFactor.intValue() > 0,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".replicationFactor").concat(gtZeroWarning));

  return new NewTopic(topicName, numPartitions, replicationFactor).configs(configs);
}

Source File: KafkaTopicClientImplTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0

6 votes

@Test
public void shouldSetTopicCleanupPolicyToCompact() throws InterruptedException,
                                                          ExecutionException {
  expect(adminClient.listTopics()).andReturn(getEmptyListTopicResult());

  // Verify that the new topic configuration being passed to the admin client is what we expect.
  NewTopic newTopic = new NewTopic(topicName1, 1, (short) 1);
  newTopic.configs(Collections.singletonMap("cleanup.policy", "compact"));
  expect(adminClient.createTopics(singleNewTopic(newTopic))).andReturn(getCreateTopicsResult());
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  kafkaTopicClient.createTopic(topicName1,
                               1,
                               (short) 1,
                               Collections.singletonMap("cleanup.policy", "compact"));
  verify(adminClient);
}

Source File: KafkaTopicClientImplTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0

6 votes

private static Collection<NewTopic> singleNewTopic(final NewTopic expected) {
  class NewTopicsMatcher implements IArgumentMatcher {
    @SuppressWarnings("unchecked")
    @Override
    public boolean matches(final Object argument) {
      final Collection<NewTopic> newTopics = (Collection<NewTopic>) argument;
      if (newTopics.size() != 1) {
        return false;
      }

      final NewTopic actual = newTopics.iterator().next();
      return Objects.equals(actual.name(), expected.name())
             && Objects.equals(actual.replicationFactor(), expected.replicationFactor())
             && Objects.equals(actual.numPartitions(), expected.numPartitions())
             && Objects.equals(actual.configs(), expected.configs());
    }

    @Override
    public void appendTo(final StringBuffer buffer) {
      buffer.append("{NewTopic").append(expected).append("}");
    }
  }

  EasyMock.reportMatcher(new NewTopicsMatcher());
  return null;
}

org.apache.kafka.clients.admin.NewTopic Java Examples