Java Code Examples for org.apache.flink.util.NetUtils#getAvailablePort()

@Test(timeout = 10000L)
public void testLogNotFoundHandling() throws Throwable {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		try {
			CompletableFuture<TransientBlobKey> logFuture =
				tmGateway.requestFileUpload(FileType.LOG, timeout);
			logFuture.get();
		} catch (Exception e) {
			assertThat(e.getMessage(), containsString("The file LOG does not exist on the TaskExecutor."));
		}
	}
}
 

/**
 * Creates a GatewayServer run in a daemon thread.
 *
 * @return The created GatewayServer
 */
static GatewayServer startGatewayServer() throws ExecutionException, InterruptedException {
	CompletableFuture<GatewayServer> gatewayServerFuture = new CompletableFuture<>();
	Thread thread = new Thread(() -> {
		try {
			int freePort = NetUtils.getAvailablePort();
			GatewayServer server = new GatewayServer.GatewayServerBuilder()
				.gateway(new Gateway(new ConcurrentHashMap<String, Object>(), new CallbackClient(freePort)))
				.javaPort(0)
				.build();
			resetCallbackClientExecutorService(server);
			gatewayServerFuture.complete(server);
			server.start(true);
		} catch (Throwable e) {
			gatewayServerFuture.completeExceptionally(e);
		}
	});
	thread.setName("py4j-gateway");
	thread.setDaemon(true);
	thread.start();
	thread.join();
	return gatewayServerFuture.get();
}
 

@Before
public void setUp() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.generateSequence(1, 1000).output(new DiscardingOutputFormat<Long>());

	Plan plan = env.createProgramPlan();
	JobWithJars jobWithJars = new JobWithJars(plan, Collections.<URL>emptyList(),  Collections.<URL>emptyList());

	program = mock(PackagedProgram.class);
	when(program.getPlanWithJars()).thenReturn(jobWithJars);

	final int freePort = NetUtils.getAvailablePort();
	config = new Configuration();
	config.setString(JobManagerOptions.ADDRESS, "localhost");
	config.setInteger(JobManagerOptions.PORT, freePort);
	config.setString(AkkaOptions.ASK_TIMEOUT, AkkaOptions.ASK_TIMEOUT.defaultValue());
}
 

@Before
public void setUp() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.generateSequence(1, 1000).output(new DiscardingOutputFormat<>());
	plan = env.createProgramPlan();

	final int freePort = NetUtils.getAvailablePort();
	config = new Configuration();
	config.setString(JobManagerOptions.ADDRESS, "localhost");
	config.setInteger(JobManagerOptions.PORT, freePort);
	config.setString(AkkaOptions.ASK_TIMEOUT, AkkaOptions.ASK_TIMEOUT.defaultValue());
}
 

private static NettyConfig createNettyConfig(Configuration config) {
	return new NettyConfig(
			InetAddress.getLoopbackAddress(),
			NetUtils.getAvailablePort(),
			NettyTestUtil.DEFAULT_SEGMENT_SIZE,
			1,
			config);
}
 

static NettyConfig createConfig(int segmentSize, Configuration config) throws Exception {
	checkArgument(segmentSize > 0);
	checkNotNull(config);

	return new NettyConfig(
			InetAddress.getLocalHost(),
			NetUtils.getAvailablePort(),
			segmentSize,
			1,
			config);
}
 

/**
 * Give the BlobServer a choice of three ports, where two of them
 * are allocated.
 */
@Test
public void testOnePortAvailable() throws IOException {
	int numAllocated = 2;
	ServerSocket[] sockets = new ServerSocket[numAllocated];
	for (int i = 0; i < numAllocated; i++) {
		try {
			sockets[i] = new ServerSocket(0);
		} catch (IOException e) {
			e.printStackTrace();
			Assert.fail("An exception was thrown while preparing the test " + e.getMessage());
		}
	}
	int availablePort = NetUtils.getAvailablePort();
	Configuration conf = new Configuration();
	conf.setString(BlobServerOptions.PORT, sockets[0].getLocalPort() + "," + sockets[1].getLocalPort() + "," + availablePort);
	conf.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	// this thing is going to throw an exception
	try {
		BlobServer server = new BlobServer(conf, new VoidBlobStore());
		server.start();
		Assert.assertEquals(availablePort, server.getPort());
		server.close();
	} finally {
		for (int i = 0; i < numAllocated; ++i) {
			sockets[i].close();
		}
	}
}
 

/**
 * Tests that repeated remote {@link PartitionNotFoundException}s ultimately fail the receiver.
 */
@Test(timeout = 10000L)
public void testRemotePartitionNotFound() throws Exception {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);

	// Remote location (on the same TM though) for the partition
	NettyShuffleDescriptor sdd =
		NettyShuffleDescriptorBuilder.newBuilder().setDataPort(dataPort).buildRemote();
	TaskDeploymentDescriptor tdd = createReceiver(sdd);
	ExecutionAttemptID eid = tdd.getExecutionAttemptId();

	final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> taskFailedFuture = new CompletableFuture<>();

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setSlotSize(2)
			.addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture)
			.addTaskManagerActionListener(eid, ExecutionState.FAILED, taskFailedFuture)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.useRealNonMockShuffleEnvironment()
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
		taskRunningFuture.get();

		taskFailedFuture.get();
		assertThat(taskSlotTable.getTask(eid).getFailureCause(), instanceOf(PartitionNotFoundException.class));
	}
}
 

/**
 * Tests that repeated remote {@link PartitionNotFoundException}s ultimately fail the receiver.
 */
@Test(timeout = TEST_TIMEOUT)
public void testRemotePartitionNotFound() throws Exception {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);

	// Remote location (on the same TM though) for the partition
	NettyShuffleDescriptor sdd =
		NettyShuffleDescriptorBuilder.newBuilder().setDataPort(dataPort).buildRemote();
	TaskDeploymentDescriptor tdd = createReceiver(sdd);
	ExecutionAttemptID eid = tdd.getExecutionAttemptId();

	final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> taskFailedFuture = new CompletableFuture<>();

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setSlotSize(2)
			.addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture)
			.addTaskManagerActionListener(eid, ExecutionState.FAILED, taskFailedFuture)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.useRealNonMockShuffleEnvironment()
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
		taskRunningFuture.get();

		taskFailedFuture.get();
		assertThat(taskSlotTable.getTask(eid).getFailureCause(), instanceOf(PartitionNotFoundException.class));
	}
}
 

private static NettyConfig createNettyConfig(Configuration config) {
	return new NettyConfig(
			InetAddress.getLoopbackAddress(),
			NetUtils.getAvailablePort(),
			NettyTestUtil.DEFAULT_SEGMENT_SIZE,
			1,
			config);
}
 

/**
 * Give the BlobServer a choice of three ports, where two of them
 * are allocated.
 */
@Test
public void testOnePortAvailable() throws IOException {
	int numAllocated = 2;
	ServerSocket[] sockets = new ServerSocket[numAllocated];
	for (int i = 0; i < numAllocated; i++) {
		try {
			sockets[i] = new ServerSocket(0);
		} catch (IOException e) {
			e.printStackTrace();
			Assert.fail("An exception was thrown while preparing the test " + e.getMessage());
		}
	}
	int availablePort = NetUtils.getAvailablePort();
	Configuration conf = new Configuration();
	conf.setString(BlobServerOptions.PORT, sockets[0].getLocalPort() + "," + sockets[1].getLocalPort() + "," + availablePort);
	conf.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	// this thing is going to throw an exception
	try {
		BlobServer server = new BlobServer(conf, new VoidBlobStore());
		server.start();
		Assert.assertEquals(availablePort, server.getPort());
		server.close();
	} finally {
		for (int i = 0; i < numAllocated; ++i) {
			sockets[i].close();
		}
	}
}
 

static NettyConfig createConfig(int segmentSize, Configuration config) throws Exception {
	checkArgument(segmentSize > 0);
	checkNotNull(config);

	return new NettyConfig(
			InetAddress.getLocalHost(),
			NetUtils.getAvailablePort(),
			segmentSize,
			1,
			config);
}
 

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

@Test
public void testLogNotFoundHandling() throws Exception {

	new JavaTestKit(system){{

		// we require a JobManager so that the BlobService is also started
		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		try {

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
				new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			final int dataPort = NetUtils.getAvailablePort();
			Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.DATA_PORT, dataPort);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
			config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			taskManager = TestingUtils.createTaskManager(
				system,
				highAvailabilityServices,
				config,
				false,
				true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			new Within(d) {
				@Override
				protected void run() {
					Future<Object> logFuture = tm.ask(TaskManagerMessages.getRequestTaskManagerLog(), timeout);
					try {
						Await.result(logFuture, timeout);
						Assert.fail();
					} catch (Exception e) {
						Assert.assertTrue(e.getMessage().startsWith("TaskManager log files are unavailable. Log file could not be found at"));
					}
				}
			};
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
 

/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}