kafka.server.KafkaServer#startup

Source File: KafkaNotification.java From incubator-atlas with Apache License 2.0

6 votes

private void startKafka() throws IOException, URISyntaxException {
    String kafkaValue = properties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
    LOG.debug("Starting kafka at {}", kafkaValue);
    URL kafkaAddress = getURL(kafkaValue);

    Properties brokerConfig = properties;
    brokerConfig.setProperty("broker.id", "1");
    brokerConfig.setProperty("host.name", kafkaAddress.getHost());
    brokerConfig.setProperty("port", String.valueOf(kafkaAddress.getPort()));
    brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath());
    brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));

    kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(),
            Option.apply(this.getClass().getName()));
    kafkaServer.startup();
    LOG.debug("Embedded kafka server started with broker config {}", brokerConfig);
}

Source File: EmbeddedKafkaBroker.java From ameliant-tools with Apache License 2.0

6 votes

@Override
protected void before() throws Throwable {
    logDirectory = tempDir(perTest("kafka-log"));
    Properties properties = brokerDefinition.getProperties();
    properties.setProperty(KafkaConfig.LogDirProp(), logDirectory.getCanonicalPath());
    kafkaServer = new KafkaServer(new KafkaConfig(properties),
            SystemTime$.MODULE$, Some$.MODULE$.apply("kafkaServer"));
    kafkaServer.startup();

    List<TopicDefinition> topicDefinitions = brokerDefinition.getTopicDefinitions();
    if (!topicDefinitions.isEmpty()) {
        ZkUtils zkUtils = ZkUtils.apply(brokerDefinition.getZookeeperConnect(), 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        for (TopicDefinition topicDefinition : topicDefinitions) {
            String name = topicDefinition.getName();
            log.info("Creating topic {}", name);
            AdminUtils.createTopic(zkUtils,
                    name,
                    topicDefinition.getPartitions(),
                    topicDefinition.getReplicationFactor(),
                    topicDefinition.getProperties());
        }
    }
}

Source File: KafkaStormIntegrationTest.java From incubator-retired-pirk with Apache License 2.0

6 votes

private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}

Source File: EmbeddedKafkaServer.java From atlas with Apache License 2.0

6 votes

private void startKafka() throws IOException, URISyntaxException {
    String kafkaValue = properties.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);

    LOG.info("Starting kafka at {}", kafkaValue);

    URL        kafkaAddress = getURL(kafkaValue);
    Properties brokerConfig = properties;

    brokerConfig.setProperty("broker.id", "1");
    brokerConfig.setProperty("host.name", kafkaAddress.getHost());
    brokerConfig.setProperty("port", String.valueOf(kafkaAddress.getPort()));
    brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath());
    brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));

    List<KafkaMetricsReporter>   metrics          = new ArrayList<>();
    Buffer<KafkaMetricsReporter> metricsReporters = scala.collection.JavaConversions.asScalaBuffer(metrics);

    kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(), Option.apply(this.getClass().getName()), metricsReporters);

    kafkaServer.startup();

    LOG.info("Embedded kafka server started with broker config {}", brokerConfig);
}

Source File: EmbeddedKafkaServer.java From twill with Apache License 2.0

5 votes

@Override
protected void startUp() throws Exception {
  int tries = 0;
  do {
    KafkaConfig kafkaConfig = createKafkaConfig(properties);
    KafkaServer kafkaServer = createKafkaServer(kafkaConfig);
    try {
      kafkaServer.startup();
      server = kafkaServer;
    } catch (Exception e) {
      kafkaServer.shutdown();
      kafkaServer.awaitShutdown();

      Throwable rootCause = Throwables.getRootCause(e);
      if (rootCause instanceof ZkTimeoutException) {
        // Potentially caused by race condition bug described in TWILL-139.
        LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause);
      } else if (rootCause instanceof BindException) {
        LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause);
      } else {
        throw e;
      }

      // Do a random sleep of < 200ms
      TimeUnit.MILLISECONDS.sleep(new Random().nextInt(200) + 1L);
    }
  } while (server == null && ++tries < startTimeoutRetries);

  if (server == null) {
    throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts.");
  }
}

Source File: EmbeddedKafka.java From kafka-pubsub-emulator with Apache License 2.0

5 votes

public EmbeddedKafka create(Integer nodeId) throws Exception {
  int port = INIT_PORT + nodeId;
  String logDir = temporaryFolder.newFolder().getAbsolutePath();

  KafkaConfig kafkaConfig = getKafkaConfig(nodeId, port, logDir);

  kafkaServer =
      new KafkaServer(kafkaConfig, Time.SYSTEM, Option.empty(), asScalaBuffer(new ArrayList<>()));
  kafkaServer.startup();
  return this;
}

Source File: EmbeddedKafkaCluster.java From logback-kafka-appender with Apache License 2.0

4 votes

private KafkaServer startBroker(Map<String, String> props) {
    KafkaServer server = new KafkaServer(new KafkaConfig(props), Time.SYSTEM,
            Some.apply("embedded-kafka-cluster"), Vector$.MODULE$.<KafkaMetricsReporter>empty());
    server.startup();
    return server;
}

Source File: KafkaTestEnvironmentImpl.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}

Source File: EmbeddedKafkaCluster.java From brooklin with BSD 2-Clause "Simplified" License

4 votes

private KafkaServer startBroker(Properties props) {
  KafkaServer server = new KafkaServer(KafkaConfig.fromProps(props), new SystemTime(),
      scala.Option.apply(""), scala.collection.JavaConversions.asScalaBuffer(Collections.emptyList()));
  server.startup();
  return server;
}

Source File: MiniKafkaCluster.java From AthenaX with Apache License 2.0

4 votes

public void start() {
  for (KafkaServer s : kafkaServer) {
    s.startup();
  }
}

Source File: KafkaBrokerTestHarness.java From common-kafka with Apache License 2.0

4 votes

private static KafkaServer startBroker(KafkaConfig config) {
    KafkaServer server = new KafkaServer(config, Time.SYSTEM, Option.empty(),
            new scala.collection.mutable.MutableList<>());
    server.startup();
    return server;
}