Java Code Examples for kafka.admin.AdminUtils#createTopic()

public void createTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = new Properties(defaultTopicConfig);
  topicConfig.setProperty(LEADER_THROTTLED_REPLICAS, WILDCARD);
  topicConfig.setProperty(FOLLOWER_THROTTLED_REPLICAS, WILDCARD);
  RoadType roadType = ofNullable(road.getType()).orElse(RoadType.NORMAL);
  switch (roadType) {
  case NORMAL:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
    break;
  case COMPACT:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
    break;
  default:
    throw new KafkaException("Unhandled road type \"" + road.getType().name() + "\"");
  }
  AdminUtils
      .createTopic(zkUtils, road.getTopicName(), partitions, replicationFactor, topicConfig, DEFAULT_RACK_AWARE_MODE);
  log.info("Created {} topic {}", roadType, road.getTopicName());
}
 

@Test
public void createTopicInTestEnv() {
	String ZOOKEEPER_CONNECT = "";
	ZkClient zkClient = new ZkClient(new ZkConnection(ZOOKEEPER_CONNECT));
	ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(ZOOKEEPER_CONNECT), false);
	zkClient.setZkSerializer(new ZKStringSerializer());
	int partition = 1;
	int replication = 1;
	String topic = String.format("kafka.test_create_topic_p%s_r%s", partition, replication);
	if (AdminUtils.topicExists(zkUtils, topic)) {
		TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
		System.out.println(topicMetadata);
		AdminUtils.deleteTopic(zkUtils, topic);
	}
	AdminUtils.createTopic(zkUtils, topic, partition, replication, new Properties());
}
 

@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 

/**
 * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...]
 * @param topic topic to create (if not already existing)
 * @param partitions number of topic partitions
 * @param topicProperties optional topic config properties
 */
public static void maybeCreateTopic(String zkServers,
                                    String topic,
                                    int partitions,
                                    Properties topicProperties) {
  ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false);
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      log.info("No need to create topic {} as it already exists", topic);
    } else {
      log.info("Creating topic {} with {} partition(s)", topic, partitions);
      try {
        AdminUtils.createTopic(
            zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$);
        log.info("Created topic {}", topic);
      } catch (TopicExistsException re) {
        log.info("Topic {} already exists", topic);
      }
    }
  } finally {
    zkUtils.close();
  }
}
 

@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics        
    AdminUtils.createTopic(zkUtils, TOPIC_OOS, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 

@Override
protected void startAction() throws Exception {
  ZkClient zookeeperClient =
      new ZkClient(
          KafkaFlags.getInstance().zookeeperIp, 15000, 10000, ZKStringSerializer$.MODULE$);
  ZkUtils zookeeperUtils =
      new ZkUtils(zookeeperClient, new ZkConnection(KafkaFlags.getInstance().zookeeperIp), false);
  try {
    deleteTopic(zookeeperUtils);
    AdminUtils.createTopic(
        zookeeperUtils,
        topic,
        KafkaFlags.getInstance().partitions,
        KafkaFlags.getInstance().replicationFactor,
        AdminUtils.createTopic$default$5(),
        AdminUtils.createTopic$default$6());
    log.info("Created topic " + topic + ".");
  } finally {
    zookeeperClient.close();
  }
}
 

/**
 * Create Kafka topic based on the destination connection string, if it does not already exist.
 * @param connectionString connection string from which to obtain topic name
 * @param numberOfPartitions number of partitions
 * @param topicConfig topic config to use for topic creation
 */
public void createTopic(String connectionString, int numberOfPartitions, Properties topicConfig) {
  Validate.notNull(connectionString, "destination should not be null");
  Validate.notNull(topicConfig, "topicConfig should not be null");
  Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present");

  String topicName = KafkaTransportProviderUtils.getTopicName(connectionString);
  populateTopicConfig(topicConfig);
  try {
    // Create only if it doesn't exist.
    if (!AdminUtils.topicExists(_zkUtils.get(), topicName)) {
      int replicationFactor = Integer.parseInt(topicConfig.getProperty("replicationFactor", DEFAULT_REPLICATION_FACTOR));
      LOG.info("Creating topic with name {} partitions={} with properties {}", topicName, numberOfPartitions,
              topicConfig);

      AdminUtils.createTopic(_zkUtils.get(), topicName, numberOfPartitions, replicationFactor, topicConfig, RackAwareMode.Disabled$.MODULE$);
    } else {
      LOG.warn("Topic with name {} already exists", topicName);
    }
  } catch (Throwable e) {
    LOG.error("Creating topic {} failed with exception {}", topicName, e);
    throw e;
  }
}
 

private void provisionTopic(String topicName,Config config) {
String zooKeeperPropKey = KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER;
if(!config.hasPath(zooKeeperPropKey)) {
 log.debug("Topic "+topicName+" is configured without the partition and replication");
 return;
}
String zookeeperConnect = config.getString(zooKeeperPropKey);
int sessionTimeoutMs = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.ZOOKEEPER_SESSION_TIMEOUT, KafkaWriterConfigurationKeys.ZOOKEEPER_SESSION_TIMEOUT_DEFAULT);
int connectionTimeoutMs = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.ZOOKEEPER_CONNECTION_TIMEOUT, KafkaWriterConfigurationKeys.ZOOKEEPER_CONNECTION_TIMEOUT_DEFAULT);
// Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
// createTopic() will only seem to work (it will return without error).  The topic will exist in
// only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
// topic.
ZkClient zkClient = new ZkClient(zookeeperConnect, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
// Security for Kafka was added in Kafka 0.9.0.0
ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect), false);
int partitions = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.PARTITION_COUNT, KafkaWriterConfigurationKeys.PARTITION_COUNT_DEFAULT);
int replication = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.REPLICATION_COUNT, KafkaWriterConfigurationKeys.PARTITION_COUNT_DEFAULT);
Properties topicConfig = new Properties();
if(AdminUtils.topicExists(zkUtils, topicName)) {
log.debug("Topic"+topicName+" already Exists with replication: "+replication+" and partitions :"+partitions);
   return;
}
try {
   AdminUtils.createTopic(zkUtils, topicName, partitions, replication, topicConfig);
} catch (RuntimeException e) {
   throw new RuntimeException(e);
}
   log.info("Created Topic "+topicName+" with replication: "+replication+" and partitions :"+partitions);
}
 

private void createKafkaTopics() {
    log.info("--------------------------------------------------------------------------");
    log.info("-----------------------Begin to create Kafka topics-----------------------");

    ZkUtils zkUtils = ZkUtils.apply(zkNodes + ":2181", 30000, 30000, JaasUtils.isZkSecurityEnabled());

    if (!AdminUtils.topicExists(zkUtils, "async-task-topic")) {
        AdminUtils.createTopic(zkUtils, "async-task-topic", 1, 1,  new Properties(), new RackAwareMode.Enforced$());
    }

    zkUtils.close();

    log.info("-----------------------Kafka topics created-------------------------------");
    log.info("--------------------------------------------------------------------------");
}
 

public void createTopic(String topicName, int numOfReplication, int numPartitions) throws Exception {
  // Create a ZooKeeper client
  int sessionTimeoutMs = 1000;
  int connectionTimeoutMs = 1000;
  ZkClient zkClient = new ZkClient(zkConnect, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
  // Create a topic named "myTopic" with 8 partitions and a replication factor of 3
  Properties topicConfig = new Properties();
  AdminUtils.createTopic(zkClient, topicName, numPartitions, numOfReplication, topicConfig);
  Thread.sleep(3000);
  zkClient.close();
}
 

@VisibleForTesting
protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
    int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1);
    int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1);
    AdminUtils.createTopic(zkUtils, topicName,  numPartitions, numReplicas,
            new Properties(), RackAwareMode.Enforced$.MODULE$);
    LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas);
}
 

/**
 * Utility method to provide additional unique topics if they are required.
 * @param topicName - The Kafka topic to create.
 */
public void createTopic(final String topicName) {
    // Setup Kafka.
    ZkUtils zkUtils = null;
    try {
        logger.info("Creating Kafka Topic: '{}'", topicName);
        zkUtils = ZkUtils.apply(new ZkClient(kafkaInstance.getZookeeperConnect(), 30000, 30000, ZKStringSerializer$.MODULE$), false);
        AdminUtils.createTopic(zkUtils, topicName, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 

public void createTopic(String topic, int partition, int replication) {
	ZkClient zkClient = new ZkClient(zkServer.getConnection());
	ZkUtils zkUtils = new ZkUtils(zkClient, zkServer.getConnection(), false);
	zkClient.setZkSerializer(new ZKStringSerializer());
	AdminUtils.createTopic(zkUtils, topic, partition, replication, new Properties());
	zkClient.close();
}
 

@VisibleForTesting
protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
    int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1);
    int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1);
    AdminUtils.createTopic(zkUtils, topicName,  numPartitions, numReplicas,
            new Properties(), RackAwareMode.Enforced$.MODULE$);
    LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas);
}
 

@Test
public void testUserManagedDestination() throws Exception {
  int numberOfPartitions = 2;
  String destinationTopic = "userManaged_" + UUID.randomUUID().toString();
  _datastreamCluster =
      initializeTestDatastreamServerWithFileConnector(1, BROADCAST_STRATEGY_FACTORY, numberOfPartitions);
  int totalEvents = 10;
  _datastreamCluster.startup();

  Path tempFile1 = Files.createTempFile("testFile1", "");
  String fileName1 = tempFile1.toAbsolutePath().toString();

  ZkClient zkClient = new ZkClient(_datastreamCluster.getZkConnection());
  ZkConnection zkConnection = new ZkConnection(_datastreamCluster.getZkConnection());
  ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
  AdminUtils.createTopic(zkUtils, destinationTopic, numberOfPartitions, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

  Datastream fileDatastream1 = createFileDatastream(fileName1, destinationTopic, 2);
  Assert.assertEquals((int) fileDatastream1.getDestination().getPartitions(), 2);

  Collection<String> eventsWritten1 = TestUtils.generateStrings(totalEvents);
  FileUtils.writeLines(new File(fileName1), eventsWritten1);

  Collection<String> eventsReceived1 = readFileDatastreamEvents(fileDatastream1, 0, 4);
  Collection<String> eventsReceived2 = readFileDatastreamEvents(fileDatastream1, 1, 6);
  eventsReceived1.addAll(eventsReceived2);

  LOG.info("Events Received " + eventsReceived1);
  LOG.info("Events Written to file " + eventsWritten1);

  Assert.assertTrue(eventsReceived1.containsAll(eventsWritten1));
}
 

@Override
public void open(String txId) {
	// Add topic configuration here
       Properties topicConfig = new Properties();

       AdminUtils.createTopic(zkUtils, txId, zooPartitions, zooReplication, topicConfig, RackAwareMode.Enforced$.MODULE$);
}
 

public void createTopics(String... topics) {
    for (String topic : topics) {
        AdminUtils.createTopic(getZkClient(), topic, 2, 1, new Properties());
    }
}
 

public void makeSureTopic(String zkConnectionString) {
    ZkClient zkClient = new ZkClient(zkConnectionString, 10000, 10000, ZKStringSerializer$.MODULE$);
    Properties topicConfiguration = new Properties();
    AdminUtils.createTopic(zkClient, TOPIC, 1, 1, topicConfiguration);
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
    //System.setProperty("hadoop.home.dir", "C:\\Users\\b60\\Desktop\\hadoop-2.6.0\\hadoop-2.6.0");

    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    String zkConnect = ZOOKEEPER_HOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Kafka
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + KAFKA_HOST + ":" + KAFKA_PORT);
    brokerProps.setProperty("offsets.topic.replication.factor" , "1");
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topic
    AdminUtils.createTopic(zkUtils, TOPIC_PROCESS_INSTANCE, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_ACTIVITY_INSTANCE, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_VARIABLE_UPDATE, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

    // setup producer
    Properties producerProps = new Properties();
    producerProps.setProperty("bootstrap.servers", KAFKA_HOST + ":" + KAFKA_PORT);
    producerProps.setProperty("key.serializer","org.apache.kafka.common.serialization.IntegerSerializer");
    producerProps.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    KafkaProducer<Integer, String> producer = new KafkaProducer<>(producerProps);

    //fill in test data
    try (Stream<String> stream = Files.lines(Paths.get(FILE_STREAM_INPUT_PROCESS_INSTANCE))) {
        stream.forEach(l -> producer.send(new ProducerRecord<>(TOPIC_PROCESS_INSTANCE, 0, 0, l)));
    }

    try (Stream<String> stream = Files.lines(Paths.get(FILE_STREAM_INPUT_ACTIVITY_INSTANCE))) {
        stream.forEach(l -> producer.send(new ProducerRecord<>(TOPIC_ACTIVITY_INSTANCE, 0, 0, l)));
    }

    try (Stream<String> stream = Files.lines(Paths.get(FILE_STREAM_INPUT_VARIABLE_UPDATE))) {
        stream.forEach(l -> producer.send(new ProducerRecord<>(TOPIC_VARIABLE_UPDATE, 0, 0, l)));
    }
}
 

static void createKafkaTopic(String name, int partitions, int replicas) {
	AdminUtils.createTopic(zkClient, name, partitions, replicas, new Properties());
}