Java Code Examples for kafka.zk.KafkaZkClient#apply()

public void createTopic(String topicName, int numPartitions) {
    // setup

    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: CreateTopic " + topicName);
        adminZkClient.createTopic(topicName,numPartitions, 1,new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        zkClient.close();
    }

}
 

/**
 * @return All topic names
 */
public List<String> listTopics() {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: ListTopics ");

            return JavaConversions.asJavaCollection(adminZkClient.getAllTopicConfigs().keys())
                    .stream()
                    .collect(Collectors.toList());

    } finally {
        zkClient.close();
    }
}
 

/**
 * Delete a topic.
 *
 * @param topicName The name of the topic to delete
 */
public void deleteTopic(String topicName) {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: DeleteTopic " + topicName);
        adminZkClient.deleteTopic(topicName);
    } finally {
        zkClient.close();
    }
}
 

void maybeElectLeader() throws Exception {
  if (!_preferredLeaderElectionRequested) {
    return;
  }

  try (KafkaZkClient zkClient = KafkaZkClient.apply(_zkConnect, JaasUtils.isZkSecurityEnabled(), com.linkedin.kmf.common.Utils.ZK_SESSION_TIMEOUT_MS,
      com.linkedin.kmf.common.Utils.ZK_CONNECTION_TIMEOUT_MS, Integer.MAX_VALUE, Time.SYSTEM, METRIC_GROUP_NAME, "SessionExpireListener", null)) {
    if (!zkClient.reassignPartitionsInProgress()) {
      List<TopicPartitionInfo> partitionInfoList = _adminClient
          .describeTopics(Collections.singleton(_topic)).all().get().get(_topic).partitions();
      LOGGER.info(
          "MultiClusterTopicManagementService will trigger requested preferred leader election for the"
              + " topic {} in cluster.", _topic);
      triggerPreferredLeaderElection(partitionInfoList, _topic);
      _preferredLeaderElectionRequested = false;
    }
  }
}
 

public static void createTopic(String kafkaTopic, int numOfPartitions, String zkStr, String replicatorFactor) {
  // TopicCommand.main() will call System.exit() finally, which will break maven-surefire-plugin
  try {
    String[] args = new String[]{"--create", "--zookeeper", zkStr, "--replication-factor", replicatorFactor,
        "--partitions", String.valueOf(numOfPartitions), "--topic", kafkaTopic};
    KafkaZkClient zkClient = KafkaZkClient
        .apply(zkStr, false, 3000, 3000, Integer.MAX_VALUE, Time.SYSTEM, "kafka.server",
            "SessionExpireListener");
    TopicCommand.TopicCommandOptions opts = new TopicCommand.TopicCommandOptions(args);
    TopicCommand.createTopic(zkClient, opts);
  } catch (TopicExistsException e) {
    // Catch TopicExistsException otherwise it will break maven-surefire-plugin
    System.out.println("Topic already existed");
  }
}
 

/** Creates and starts the cluster. */
public void start() throws Exception {
  LOGGER.debug("Initiating embedded Kafka cluster startup");
  LOGGER.debug("Starting a ZooKeeper instance...");
  zookeeper = new ZooKeeperEmbedded();
  LOGGER.debug("ZooKeeper instance is running at {}", zookeeper.connectString());

  zkClient =
      KafkaZkClient.apply(
          zookeeper.connectString(),
          JaasUtils.isZkSecurityEnabled(),
          30000,
          30000,
          1000,
          new MockTime(),
          "kafka.server",
          "SessionExpireListener");

  final Properties effectiveBrokerConfig = effectiveBrokerConfigFrom(brokerConfig, zookeeper);
  LOGGER.debug(
      "Starting a Kafka instance on port {} ...",
      effectiveBrokerConfig.getProperty(KafkaConfig$.MODULE$.PortProp()));
  broker = new KafkaEmbedded(effectiveBrokerConfig, new MockTime());
  LOGGER.debug(
      "Kafka instance is running at {}, connected to ZooKeeper at {}",
      broker.brokerList(),
      broker.zookeeperConnect());

  final Properties schemaRegistryProps = new Properties();
  schemaRegistryProps.put(
      SchemaRegistryConfig.KAFKASTORE_TIMEOUT_CONFIG, KAFKASTORE_OPERATION_TIMEOUT_MS);
  schemaRegistryProps.put(SchemaRegistryConfig.DEBUG_CONFIG, KAFKASTORE_DEBUG);
  schemaRegistryProps.put(
      SchemaRegistryConfig.KAFKASTORE_INIT_TIMEOUT_CONFIG, KAFKASTORE_INIT_TIMEOUT);

  schemaRegistry =
      new RestApp(
          0,
          zookeeperConnect(),
          KAFKA_SCHEMAS_TOPIC,
          AVRO_COMPATIBILITY_TYPE,
          schemaRegistryProps);
  schemaRegistry.start();

  LOGGER.debug("Starting a Connect standalone instance...");
  connect = new ConnectStandalone(connectWorkerConfig());
  connect.start();
  LOGGER.debug("Connect standalone instance is running at {}", connect.getConnectionString());
  running = true;
}
 

/**
 * Create an instance of KafkaZkClient with security disabled.
 * Name of the underlying {@link kafka.zookeeper.ZooKeeperClient} instance is derived using the combination given
 * metricGroup and metricType with a dash in between.
 *
 * @param connectString Comma separated host:port pairs, each corresponding to a zk server
 * @param metricGroup Metric group
 * @param metricType Metric type
 * @param zkSecurityEnabled True if zkSecurityEnabled, false otherwise.
 * @return A new instance of KafkaZkClient
 */
public static KafkaZkClient createKafkaZkClient(String connectString, String metricGroup, String metricType, boolean zkSecurityEnabled) {
  String zooKeeperClientName = String.format("%s-%s", metricGroup, metricType);
  return KafkaZkClient.apply(connectString, zkSecurityEnabled, ZK_SESSION_TIMEOUT, ZK_CONNECTION_TIMEOUT, Integer.MAX_VALUE,
                             new SystemTime(), metricGroup, metricType, Option.apply(zooKeeperClientName));
}