org.springframework.kafka.config.StreamsBuilderFactoryBean Java Examples

@Override
public Object postProcessAfterInitialization(Object bean, String beanName)
		throws BeansException {
	if (bean instanceof StreamsBuilderFactoryBean) {
		// KafkaStreamsTracing is created in SleuthKafkaStreamsConfiguration above, so
		// should not be null here
		KafkaStreamsTracing kafkaStreamsTracing = this.objectProvider
				.getIfAvailable();
		((StreamsBuilderFactoryBean) bean)
				.setClientSupplier(kafkaStreamsTracing.kafkaClientSupplier());
		if (log.isDebugEnabled()) {
			log.debug(
					"StreamsBuilderFactoryBean bean is auto-configured to enable tracing.");
		}
	}
	return bean;
}
 

@Test
public void testStateStoreRetrievalRetry() {

	StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class);
	KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class);
	Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams);
	KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry();
	kafkaStreamsRegistry.registerKafkaStreams(mock);
	KafkaStreamsBinderConfigurationProperties binderConfigurationProperties =
			new KafkaStreamsBinderConfigurationProperties(new KafkaProperties());
	binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3);
	InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry,
			binderConfigurationProperties);

	QueryableStoreType<ReadOnlyKeyValueStore<Object, Object>> storeType = QueryableStoreTypes.keyValueStore();
	try {
		interactiveQueryService.getQueryableStore("foo", storeType);
	}
	catch (Exception ignored) {

	}

	Mockito.verify(mockKafkaStreams, times(3)).store("foo", storeType);
}
 

@Override
public synchronized void start() {
	if (!this.running) {
		try {
			Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue
					.getStreamsBuilderFactoryBeans();
			int n = 0;
			for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) {
				streamsBuilderFactoryBean.start();
				this.kafkaStreamsRegistry.registerKafkaStreams(streamsBuilderFactoryBean);
				if (this.listener != null) {
					this.listener.streamsAdded("streams." + n++, streamsBuilderFactoryBean.getKafkaStreams());
				}
			}
			if (this.kafkaStreamsBinderMetrics != null) {
				this.kafkaStreamsBinderMetrics.addMetrics(streamsBuilderFactoryBeans);
			}
			this.running = true;
		}
		catch (Exception ex) {
			throw new KafkaException("Could not start stream: ", ex);
		}
	}
}
 

@Override
public synchronized void stop() {
	if (this.running) {
		try {
			Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsBindingInformationCatalogue
					.getStreamsBuilderFactoryBeans();
			int n = 0;
			for (StreamsBuilderFactoryBean streamsBuilderFactoryBean : streamsBuilderFactoryBeans) {
				streamsBuilderFactoryBean.stop();
				if (this.listener != null) {
					this.listener.streamsRemoved("streams." + n++, streamsBuilderFactoryBean.getKafkaStreams());
				}
			}
		}
		catch (Exception ex) {
			throw new IllegalStateException(ex);
		}
		finally {
			this.running = false;
		}
	}
}
 

@Test
public void testBinderStartsSuccessfullyWhenTwoProcessorsWithSameNamesAndBindingsPresent() {
	SpringApplication app = new SpringApplication(
			MultiProcessorsWithSameNameAndBindingTests.WordCountProcessorApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=words",
			"--spring.cloud.stream.bindings.input-1.destination=words",
			"--spring.cloud.stream.bindings.output.destination=counts",
			"--spring.cloud.stream.bindings.output.contentType=application/json",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString())) {
		StreamsBuilderFactoryBean streamsBuilderFactoryBean1 = context
				.getBean("&stream-builder-Foo-process", StreamsBuilderFactoryBean.class);
		assertThat(streamsBuilderFactoryBean1).isNotNull();
		StreamsBuilderFactoryBean streamsBuilderFactoryBean2 = context
				.getBean("&stream-builder-Bar-process", StreamsBuilderFactoryBean.class);
		assertThat(streamsBuilderFactoryBean2).isNotNull();
	}
}
 

public StreamsBuilderFactoryBean streamsBuilderFactoryBean(String applicationId) {
	final Optional<StreamsBuilderFactoryBean> first = this.streamsBuilderFactoryBeanMap.values()
			.stream()
			.filter(streamsBuilderFactoryBean -> streamsBuilderFactoryBean
					.getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG)
					.equals(applicationId))
			.findFirst();
	return first.orElse(null);
}
 

@ReadOperation
public String kafkaStreamsTopology() {
	final List<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans = this.kafkaStreamsRegistry.streamsBuilderFactoryBeans();
	final StringBuilder topologyDescription = new StringBuilder();
	streamsBuilderFactoryBeans.stream()
			.forEach(streamsBuilderFactoryBean ->
					topologyDescription.append(streamsBuilderFactoryBean.getTopology().describe().toString()));
	return topologyDescription.toString();
}
 

@ReadOperation
public String kafkaStreamsTopology(@Selector String applicationId) {
	if (!StringUtils.isEmpty(applicationId)) {
		final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamsBuilderFactoryBean(applicationId);
		if (streamsBuilderFactoryBean != null) {
			return streamsBuilderFactoryBean.getTopology().describe().toString();
		}
		else {
			return NO_TOPOLOGY_FOUND_MSG;
		}
	}
	return NO_TOPOLOGY_FOUND_MSG;
}
 

@Test
public void testKstreamBinderWithPojoInputAndStringOuput() throws Exception {
	SpringApplication app = new SpringApplication(ProductCountApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);
	ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=foos",
			"--spring.cloud.stream.bindings.output.destination=counts-id",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.bindings.input.consumer.applicationId=ProductCountApplication-xyz",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString());
	try {
		receiveAndValidateFoo();
		// Assertions on StreamBuilderFactoryBean
		StreamsBuilderFactoryBean streamsBuilderFactoryBean = context
				.getBean("&stream-builder-ProductCountApplication-process", StreamsBuilderFactoryBean.class);
		CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean,
				"cleanupConfig", CleanupConfig.class);
		assertThat(cleanup.cleanupOnStart()).isFalse();
		assertThat(cleanup.cleanupOnStop()).isTrue();
	}
	finally {
		context.close();
	}
}
 

@Bean
StreamsBuilderFactoryBean streamsBuilderFactoryBean() {
	TestTraceStreamsBuilderFactoryBean factoryBean = new TestTraceStreamsBuilderFactoryBean();
	factoryBean.setAutoStartup(false);
	return factoryBean;
}
 

/**
 * Register the {@link KafkaStreams} object created in the application.
 * @param streamsBuilderFactoryBean {@link StreamsBuilderFactoryBean}
 */
void registerKafkaStreams(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
	final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
	this.kafkaStreams.add(kafkaStreams);
	this.streamsBuilderFactoryBeanMap.put(kafkaStreams, streamsBuilderFactoryBean);
}
 

@Bean
StreamsBuilderFactoryBean streamsBuilderFactoryBean() {
	streamsBuilderFactoryBean = mock(StreamsBuilderFactoryBean.class);
	return UserConfig.streamsBuilderFactoryBean;
}
 

@Test
public void testMultiFunctionsInSameAppWithMultiBinders() throws Exception {
	SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run(
			"--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.function.definition=process;analyze",
			"--spring.cloud.stream.bindings.process-in-0.destination=purchases",
			"--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.startOffset=latest",
			"--spring.cloud.stream.bindings.process-in-0.binder=kafka1",
			"--spring.cloud.stream.bindings.process-out-0.destination=coffee",
			"--spring.cloud.stream.bindings.process-out-0.binder=kafka1",
			"--spring.cloud.stream.bindings.process-out-1.destination=electronics",
			"--spring.cloud.stream.bindings.process-out-1.binder=kafka1",
			"--spring.cloud.stream.bindings.analyze-in-0.destination=coffee",
			"--spring.cloud.stream.bindings.analyze-in-0.binder=kafka2",
			"--spring.cloud.stream.bindings.analyze-in-1.destination=electronics",
			"--spring.cloud.stream.bindings.analyze-in-1.binder=kafka2",
			"--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=2",
			"--spring.cloud.stream.binders.kafka1.type=kstream",
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(),
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-1",
			"--spring.cloud.stream.binders.kafka1.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=process-client",
			"--spring.cloud.stream.binders.kafka2.type=kstream",
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(),
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.applicationId=my-app-2",
			"--spring.cloud.stream.binders.kafka2.environment.spring.cloud.stream.kafka.streams.binder.configuration.client.id=analyze-client")) {

		Thread.sleep(1000);
		receiveAndValidate("purchases", "coffee", "electronics");

		StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-process", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class);

		final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration();

		assertThat(processStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-1");
		assertThat(analyzeStreamsConfiguration.getProperty("application.id")).isEqualTo("my-app-2");
		assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client");
		assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client");

		Integer concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);

		concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isNull(); //thus default to 1 by Kafka Streams.
	}
}
 

@Test
public void testMultiFunctionsInSameApp() throws InterruptedException {
	SpringApplication app = new SpringApplication(MultipleFunctionsInSameApp.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run(
			"--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.function.definition=process;analyze;anotherProcess",
			"--spring.cloud.stream.bindings.process-in-0.destination=purchases",
			"--spring.cloud.stream.bindings.process-out-0.destination=coffee",
			"--spring.cloud.stream.bindings.process-out-1.destination=electronics",
			"--spring.cloud.stream.bindings.analyze-in-0.destination=coffee",
			"--spring.cloud.stream.bindings.analyze-in-1.destination=electronics",
			"--spring.cloud.stream.kafka.streams.binder.functions.analyze.applicationId=analyze-id-0",
			"--spring.cloud.stream.kafka.streams.binder.functions.process.applicationId=process-id-0",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.bindings.process-in-0.consumer.concurrency=2",
			"--spring.cloud.stream.bindings.analyze-in-0.consumer.concurrency=1",
			"--spring.cloud.stream.kafka.streams.binder.configuration.num.stream.threads=3",
			"--spring.cloud.stream.kafka.streams.binder.functions.process.configuration.client.id=process-client",
			"--spring.cloud.stream.kafka.streams.binder.functions.analyze.configuration.client.id=analyze-client",
			"--spring.cloud.stream.kafka.streams.binder.functions.anotherProcess.configuration.client.id=anotherProcess-client",
			"--spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString())) {
		receiveAndValidate("purchases", "coffee", "electronics");

		StreamsBuilderFactoryBean processStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-process", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean analyzeStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-analyze", StreamsBuilderFactoryBean.class);

		StreamsBuilderFactoryBean anotherProcessStreamsBuilderFactoryBean = context
				.getBean("&stream-builder-anotherProcess", StreamsBuilderFactoryBean.class);

		final Properties processStreamsConfiguration = processStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties analyzeStreamsConfiguration = analyzeStreamsBuilderFactoryBean.getStreamsConfiguration();
		final Properties anotherProcessStreamsConfiguration = anotherProcessStreamsBuilderFactoryBean.getStreamsConfiguration();

		assertThat(processStreamsConfiguration.getProperty("client.id")).isEqualTo("process-client");
		assertThat(analyzeStreamsConfiguration.getProperty("client.id")).isEqualTo("analyze-client");

		Integer concurrency = (Integer) processStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);
		concurrency = (Integer) analyzeStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(1);
		assertThat(anotherProcessStreamsConfiguration.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG)).isEqualTo("3");
	}
}
 

@Test
public void testSendToTombstone()
		throws Exception {
	SpringApplication app = new SpringApplication(
			WordCountProcessorApplication.class);
	app.setWebApplicationType(WebApplicationType.NONE);

	try (ConfigurableApplicationContext context = app.run("--server.port=0",
			"--spring.jmx.enabled=false",
			"--spring.cloud.stream.bindings.input.destination=words-1",
			"--spring.cloud.stream.bindings.output.destination=counts-1",
			"--spring.cloud.stream.kafka.streams.bindings.input.consumer.application-id=testKstreamWordCountWithInputBindingLevelApplicationId",
			"--spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde"
					+ "=org.apache.kafka.common.serialization.Serdes$StringSerde",
			"--spring.cloud.stream.kafka.streams.bindings.output.producer.valueSerde=org.springframework.kafka.support.serializer.JsonSerde",
			"--spring.cloud.stream.kafka.streams.timeWindow.length=5000",
			"--spring.cloud.stream.kafka.streams.timeWindow.advanceBy=0",
			"--spring.cloud.stream.bindings.input.consumer.concurrency=2",
			"--spring.cloud.stream.kafka.streams.binder.brokers="
					+ embeddedKafka.getBrokersAsString())) {
		receiveAndValidate("words-1", "counts-1");
		// Assertions on StreamBuilderFactoryBean
		StreamsBuilderFactoryBean streamsBuilderFactoryBean = context
				.getBean("&stream-builder-WordCountProcessorApplication-process", StreamsBuilderFactoryBean.class);
		KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams();
		assertThat(kafkaStreams).isNotNull();
		// Ensure that concurrency settings are mapped to number of stream task
		// threads in Kafka Streams.
		final Properties streamsConfiguration = streamsBuilderFactoryBean.getStreamsConfiguration();
		final Integer concurrency = (Integer) streamsConfiguration
				.get(StreamsConfig.NUM_STREAM_THREADS_CONFIG);
		assertThat(concurrency).isEqualTo(2);

		sendTombStoneRecordsAndVerifyGracefulHandling();

		CleanupConfig cleanup = TestUtils.getPropertyValue(streamsBuilderFactoryBean,
				"cleanupConfig", CleanupConfig.class);
		assertThat(cleanup.cleanupOnStart()).isTrue();
		assertThat(cleanup.cleanupOnStop()).isFalse();
	}
}
 

Set<StreamsBuilderFactoryBean> getStreamsBuilderFactoryBeans() {
	return this.streamsBuilderFactoryBeans;
}
 

public void addMetrics(Set<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans) {
	synchronized (KafkaStreamsBinderMetrics.this) {
		this.bindTo(streamsBuilderFactoryBeans, this.meterRegistry);
	}
}
 

public List<StreamsBuilderFactoryBean> streamsBuilderFactoryBeans() {
	return new ArrayList<>(this.streamsBuilderFactoryBeanMap.values());
}
 

/**
 * Adds a mapping for KStream -> {@link StreamsBuilderFactoryBean}.
 * @param streamsBuilderFactoryBean provides the {@link StreamsBuilderFactoryBean}
 * mapped to the KStream
 */
void addStreamBuilderFactory(StreamsBuilderFactoryBean streamsBuilderFactoryBean) {
	this.streamsBuilderFactoryBeans.add(streamsBuilderFactoryBean);
}
 

/**
 *
 * @param kafkaStreams {@link KafkaStreams} object
 * @return Corresponding {@link StreamsBuilderFactoryBean}.
 */
StreamsBuilderFactoryBean streamBuilderFactoryBean(KafkaStreams kafkaStreams) {
	return this.streamsBuilderFactoryBeanMap.get(kafkaStreams);
}