kafka.server.KafkaServerStartable Java Examples

public void startup() {
    zookeeper = new Zookeeper(zkPort, zkMaxConnections);
    zookeeper.startup();

    try {
        logDir = Files.createTempDirectory("kafka").toFile();
    } catch (IOException e) {
        throw new RuntimeException("Unable to start Kafka", e);
    }
    logDir.deleteOnExit();
    Runtime.getRuntime().addShutdownHook(new Thread(getDeleteLogDirectoryAction()));
    kafkaBrokerConfig.setProperty("zookeeper.connect", zookeeperString);
    kafkaBrokerConfig.setProperty("broker.id", "1");
    kafkaBrokerConfig.setProperty("host.name", "localhost");
    kafkaBrokerConfig.setProperty("port", Integer.toString(brokerPort));
    kafkaBrokerConfig.setProperty("log.dir", logDir.getAbsolutePath());
    kafkaBrokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));
    kafkaBrokerConfig.setProperty("delete.topic.enable", String.valueOf(true));
    kafkaBrokerConfig.setProperty("offsets.topic.replication.factor", String.valueOf(1));
    kafkaBrokerConfig.setProperty("auto.create.topics.enable", String.valueOf(false));

    broker = new KafkaServerStartable(new KafkaConfig(kafkaBrokerConfig));
    broker.startup();
}
 

public void startKafkaServer(int clusterid, int brokerid, int defaultPartitions)
{
  // before start, clean the kafka dir if it exists
  FileUtils.deleteQuietly(new File(baseDir, kafkaBaseDir));

  Properties props = new Properties();
  props.setProperty("broker.id", "" + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid][brokerid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid][brokerid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");
  if (hasMultiPartition) {
    props.setProperty("num.partitions", "2");
  } else {
    props.setProperty("num.partitions", "1");
  }

  broker[clusterid][brokerid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid][brokerid].startup();

}
 

public KafkaEmbedded(Integer kafkaPort, Integer zookeeperPort) {
    try {
        zk = new ZookeeperEmbedded(zookeeperPort);
        zk.start();

        this.port = null != kafkaPort ? kafkaPort : InstanceSpec.getRandomPort();
        logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/kafka-test-" + kafkaPort);
        FileUtils.deleteQuietly(logDir);

        KafkaConfig config = buildKafkaConfig(zk.getConnectionString());
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 

@Override
public void start() throws Exception {
    zkServer = new TestingServer(zookeeperPort, logDir);
    ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
    curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), retryPolicy);
    curatorClient.start();

    Properties props = new Properties();

    props.setProperty("zookeeper.connect", zkServer.getConnectString());
    props.setProperty("broker.id", "0");
    props.setProperty("port", "" + kafkaPort);
    props.setProperty("log.dirs", logDir.getAbsolutePath());
    props.setProperty("auto.create.topics.enable", "true");

    kafkaServer = new KafkaServerStartable(new KafkaConfig(props));
    kafkaServer.startup();
}
 

EmbeddedKafka(EmbeddedZookeeper zookeeper, int kafkaPort) throws IOException {
  this.zookeeper = checkNotNull(zookeeper, "zookeeper is null");

  this.port = kafkaPort;
  this.kafkaDataDir = Files.createTempDir();

  Properties properties = new Properties();
  properties.setProperty("broker.id", "0");
  properties.setProperty("host.name", "localhost");
  properties.setProperty("num.partitions", "2");
  properties.setProperty("log.flush.interval.messages", "10000");
  properties.setProperty("log.flush.interval.ms", "1000");
  properties.setProperty("log.retention.minutes", "60");
  properties.setProperty("log.segment.bytes", "1048576");
  properties.setProperty("auto.create.topics.enable", "false");
  properties.setProperty("zookeeper.connection.timeout.ms", "1000000");
  properties.setProperty("port", Integer.toString(port));
  properties.setProperty("log.dirs", kafkaDataDir.getAbsolutePath());
  properties.setProperty("zookeeper.connect", zookeeper.getConnectString());

  KafkaConfig config = new KafkaConfig(properties);
  this.kafka = new KafkaServerStartable(config);
}
 

private void setupKafkaBroker() {
    try {
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", "/tmp/tmp_kafka_dir");
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "1500");
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();

        // create "sensordata" topic
        ZkClient zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
        AdminUtils.createTopic(zkClient, "sensordata", 1, 1, new Properties());
        zkClient.close();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}
 

public void startup() {
  for (int i = 0; i < ports.size(); i++) {
    Integer port = ports.get(i);
    File logDir = TestUtils.constructTempDir("kafka-local");

    Properties properties = new Properties();
    properties.putAll(baseProperties);
    properties.setProperty("zookeeper.connect", zkConnection);
    properties.setProperty("broker.id", String.valueOf(i + 1));
    properties.setProperty("advertised.host.name", "127.0.0.1");
    properties.setProperty("host.name", "127.0.0.1");
    properties.setProperty("advertised.port", Integer.toString(port));
    properties.setProperty("port", Integer.toString(port));
    properties.setProperty("log.dirs", logDir.getAbsolutePath());
    properties.setProperty("offsets.topic.num.partitions", "1");
    properties.setProperty("offsets.topic.replication.factor", "1");
    properties.setProperty("log.flush.interval.messages", String.valueOf(1));

    KafkaServerStartable broker = startBroker(properties);

    brokers.add(broker);
    logDirs.add(logDir);
  }
}
 

/**
 * Starts the Kafka broker.
 *
 * @throws IOException if an error occurs during initialization
 */
public synchronized void start() throws IOException {
  log.info("Starting Kafka broker on port {}", port);

  logsDir = Files.createTempDirectory(LocalKafkaBroker.class.getSimpleName());
  logsDir.toFile().deleteOnExit();

  kafkaServer = new KafkaServerStartable(new KafkaConfig(ConfigUtils.keyValueToProperties(
      "broker.id", TEST_BROKER_ID,
      "log.dirs", logsDir.toAbsolutePath(),
      "listeners", "PLAINTEXT://:" + port,
      "zookeeper.connect", "localhost:" + zkPort,
      "message.max.bytes", 1 << 26,
      "replica.fetch.max.bytes", 1 << 26,
      "offsets.topic.replication.factor", 1
  ), false));
  kafkaServer.startup();
}
 

public LocalKafkaServer() throws IOException {

		while (new File(logDir).exists()) {
			FileUtils.deleteDirectory(new File(logDir));
		}

		Properties props = new Properties();
		props.put("broker.id", nodeId);
		props.put("port", port);
		props.put("log.dir", logDir);
		props.put("zookeeper.connect", zkConnect);
		props.put("host.name", "127.0.0.1");
		KafkaConfig conf = new KafkaConfig(props);

                zkUtils = ZkUtils.apply(props.getProperty("zookeeper.connect"),
                          30000,
                          30000,
                          JaasUtils.isZkSecurityEnabled());


		server = new KafkaServerStartable(conf);
		server.startup();
	}
 

public static KafkaServerStartable startServer(final int port, final int brokerId,
    final String zkStr, final Properties configuration) {
  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File("/tmp/kafka-" + Double.toHexString(Math.random()));
  logDir.mkdirs();
  logDir.deleteOnExit();
  kafkaDataDir = logDir.toString();

  configureKafkaPort(configuration, port);
  configureZkConnectionString(configuration, zkStr);
  configureBrokerId(configuration, brokerId);
  configureKafkaLogDirectory(configuration, logDir);
  KafkaConfig config = new KafkaConfig(configuration);
  KafkaServerStartable serverStartable = new KafkaServerStartable(config);
  serverStartable.startup();
  return serverStartable;
}
 

public void init(Properties props) {
  port = (int) props.get(PORT);
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  serverStartable = new KafkaServerStartable(new KafkaConfig(props));
  final Map<String, Object> config = new HashMap<>();
  config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
  config.put(AdminClientConfig.CLIENT_ID_CONFIG, "Kafka2AdminClient-" + UUID.randomUUID().toString());
  config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, 15000);
  adminClient = KafkaAdminClient.create(config);
}
 

public static void setupKafkaBroker2() {
    try {
        log.info("#############################################################################################");
        log.info("#################################   ZOOKEEPER 2 STARTED  ####################################");
        log.info("#############################################################################################");
        // mock zookeeper
        zkTestServer2 = new TestingServer(2182);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "1");
        props.put("host.name", "localhost");
        props.put("port", "9093");
        props.put("log.dir", kafkaLogDir2);
        props.put("zookeeper.connect", zkTestServer2.getConnectString());
        props.put("replica.socket.timeout.ms", "30000");
        props.put("delete.topic.enable", "true");
        props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE);
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer2 = new KafkaServerStartable(config);
        kafkaServer2.startup();

    } catch (Exception e) {
        log.error("Error running local Kafka broker 2", e);
    }
}
 

public static void setupKafkaBroker() {
    try {
        log.info("#############################################################################################");
        log.info("#################################   ZOOKEEPER STARTED  ######################################");
        log.info("#############################################################################################");
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", kafkaLogDir);
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "30000");
        props.put("delete.topic.enable", "true");
        props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE);
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}
 

public void init(Properties props) {
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  KafkaConfig config = new KafkaConfig(props);

  serverStartable = new KafkaServerStartable(config);
}
 

/**
 * Start local Kafka and (optionally) Zookeeper
 *
 * @param kafkaHost e.g. localhost
 * @param kafkaPort e.g. 60001
 * @param kafkaReplicationFactor e.g. 1
 * @param zookeeperHost e.g. localhost
 * @param zookeeperPort e.g. 60002
 * @param zookeeperDataDir e.g. "/tmp" or "Files.createTempDir().getAbsolutePath()"
 */
public static void start(
    String kafkaHost,
    int kafkaPort,
    short kafkaReplicationFactor,
    boolean startZookeper,
    String zookeeperHost,
    int zookeeperPort,
    String zookeeperDataDir)
    throws InterruptedException {
  if (startZookeper) {
    LocalZookeeper.start(zookeeperPort, zookeeperDataDir);
    Thread.sleep(5000);
  }
  Properties kafkaProp = new Properties();
  kafkaProp.put("zookeeper.connect", zookeeperHost + ":" + zookeeperPort);
  kafkaProp.put("host.name", kafkaHost);
  kafkaProp.put("port", kafkaPort);
  kafkaProp.put("log.dir", Files.createTempDir().getAbsolutePath());
  kafkaProp.put("offsets.topic.replication.factor", kafkaReplicationFactor);
  KafkaConfig kafkaConfig = new KafkaConfig(kafkaProp);
  server = new KafkaServerStartable(kafkaConfig);
  new Thread(server::startup).start();
  Thread.sleep(5000);
}
 

public void start() throws Exception {

        File dir = Files.createTempDir();
        String dirPath = dir.getAbsolutePath();
        System.out.println("Storing Kafka files in " + dirPath);

        String localhostKafka = System.getProperty("kafka.ip");
        String localhostZookeeper = System.getProperty("zookeeper.ip");
        String portKafka = System.getProperty("kafka.port");
        String portZookeeper = System.getProperty("zookeeper.port");

        Properties properties = new Properties();
        properties.put("broker.id", "0");
        properties.put("host.name", "localhost");
//        properties.put("host.name", localhostKafka);
        properties.put("port", KAFKA_PORT);
        properties.put("log.dir", dirPath);
        properties.put("log.flush.interval.messages", "1");
//        properties.put("zookeeper.connect", "localhost:" + ZookeeperServer.CLIENT_PORT);
        properties.put("zookeeper.connect", localhostZookeeper+":" + ZookeeperServer.CLIENT_PORT);
//        properties.put("zookeeper.connect", "port:" + ZookeeperServer.CLIENT_PORT);
//        Integer clientPort = Integer.parseInt(System.getProperty("zookeeper.port"));
//        properties.put("zookeeper.connect", "localhost:" + clientPort);
        properties.put("replica.socket.timeout.ms", "1500");
        properties.put("auto.create.topics.enable", "true");
        properties.put("num.partitions", "1");

        KafkaConfig kafkaConfig = new KafkaConfig(properties);
        kafkaServer = new KafkaServerStartable(kafkaConfig);
        kafkaServer.startup();

        TimeUnit.SECONDS.sleep(WAIT_SECONDS);
    }
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();

    this.kafkaConfig = kafkaConfig;
    this.zookeeperConfig = zookeeperConfig;

    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
    this.zkServer = new ZooKeeperServer();
}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 

public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 

public KafkaLocal(Properties kafkaProperties) throws IOException,
        InterruptedException{
  kafkaConfig = new KafkaConfig(kafkaProperties);

  //start local kafka broker
  kafka = new KafkaServerStartable(kafkaConfig);
}
 

@Override
public void run() {

    try (InputStream stream = EmbeddedKafka.class.getResourceAsStream("/kafka.properties")) {
        if (stream == null) {
            throw new IOException("kafka.properties is not available as resource");
        }
        Properties p = new Properties();
        p.load(stream);

        String dir = p.getProperty("log.dirs");
        if (dir != null) {
            File reset = new File(dir);
            if (reset.exists() && reset.isDirectory()) {
                Utility.getInstance().cleanupDir(reset);
                log.info("Clean up transient Kafka working directory at {}", dir);
            }
        }
        kafka = new KafkaServerStartable(new KafkaConfig(p));
        kafka.startup();
        Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));

    } catch (IOException e) {
        log.error("Unable to start Kafka kafka - {}", e.getMessage());
        System.exit(-1);
    }
}
 

LocalKafka(int kafkaPort, int zookeeperPort) throws Exception {
  Properties kafkaProperties = new Properties();
  kafkaProperties.setProperty("port", String.valueOf(kafkaPort));
  kafkaProperties.setProperty("zookeeper.connect", String.format("localhost:%s", zookeeperPort));
  kafkaProperties.setProperty("offsets.topic.replication.factor", "1");
  kafkaProperties.setProperty("log.dir", Files.createTempDirectory("kafka-log-").toString());
  server = new KafkaServerStartable(KafkaConfig.fromProps(kafkaProperties));
}
 

public KafkaServer(int zookeeperPort, int kafkaBrokerPort) {
    try {
        Preconditions.checkArgument(zookeeperPort > 0);
        Preconditions.checkArgument(kafkaBrokerPort > 0);
        this.zookeeperPort = zookeeperPort;
        this.brokerPort = kafkaBrokerPort;
       this.logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/hbase-cdc-kafka-" + brokerPort);

       KafkaConfig config = buildKafkaConfig(zookeeperPort);
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 

public static void stopServer(KafkaServerStartable serverStartable) {
  serverStartable.shutdown();
  if (StringUtils.isNotEmpty(kafkaDataDir)) {
    try {
      FileUtils.deleteDirectory(new File(kafkaDataDir));
    } catch (IOException e) {
      LOGGER.warn("Caught exception while stopping kafka", e);
    }
  }
}