Java Code Examples for org.springframework.cloud.stream.binder.Binder#bindConsumer()

@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicUnderPartitionedAndAutoRebalanceEnabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	Binding binding = binder.bindConsumer(testTopicName, "test", input,
			consumerProperties);
	binding.unbind();
	assertThat(invokePartitionSize(testTopicName)).isEqualTo(1);
}
 

public <T> Binding<T> doBindConsumer(T input, String inputName,
		Binder<T, ConsumerProperties, ?> binder,
		ConsumerProperties consumerProperties, String target) {
	if (this.taskScheduler == null
			|| this.bindingServiceProperties.getBindingRetryInterval() <= 0) {
		return binder.bindConsumer(target,
				this.bindingServiceProperties.getGroup(inputName), input,
				consumerProperties);
	}
	else {
		try {
			return binder.bindConsumer(target,
					this.bindingServiceProperties.getGroup(inputName), input,
					consumerProperties);
		}
		catch (RuntimeException e) {
			LateBinding<T> late = new LateBinding<T>(target,
					e.getCause() == null ? e.toString() : e.getCause().getMessage(), consumerProperties, true);
			rescheduleConsumerBinding(input, inputName, binder, consumerProperties,
					target, late, e);
			this.consumerBindings.put(inputName, Collections.singletonList(late));
			return late;
		}
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testPartitionCountIncreasedIfAutoAddPartitionsSet() throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	configurationProperties.setMinPartitionCount(6);
	configurationProperties.setAutoAddPartitions(true);
	Binder binder = getBinder(configurationProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<?> binding = binder.bindConsumer(testTopicName, "test", input,
			consumerProperties);
	binding.unbind();
	assertThat(invokePartitionSize(testTopicName)).isEqualTo(6);
}
 

@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller()
		throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(6);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(5);
	producerProperties.setPartitionKeyExpression(
			spelExpressionParser.parseExpression("payload"));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Thread.sleep(1000);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(6);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testAutoCreateTopicsEnabledSucceeds() throws Exception {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
	configurationProperties.setAutoCreateTopics(true);
	Binder binder = getBinder(configurationProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	String testTopicName = "nonexisting" + System.currentTimeMillis();
	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<?> binding = binder.bindConsumer(testTopicName, "test",
			moduleInputChannel, consumerProperties);
	binding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testConsumerDefaultDeserializer() throws Throwable {
	Binding<?> binding = null;
	try {
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
		String testTopicName = "existing" + System.currentTimeMillis();
		invokeCreateTopic(testTopicName, 5, 1);
		configurationProperties.setAutoCreateTopics(false);
		Binder binder = getBinder(configurationProperties);

		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
		DirectChannel input = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));

		binding = binder.bindConsumer(testTopicName, "test", input,
				consumerProperties);
		DirectFieldAccessor consumerAccessor = new DirectFieldAccessor(
				getKafkaConsumer(binding));
		assertThat(consumerAccessor
				.getPropertyValue("keyDeserializer") instanceof ByteArrayDeserializer)
						.isTrue();
		assertThat(consumerAccessor.getPropertyValue(
				"valueDeserializer") instanceof ByteArrayDeserializer).isTrue();
	}
	finally {
		if (binding != null) {
			binding.unbind();
		}
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testDynamicKeyExpression() throws Exception {
	Binder binder = getBinder(createConfigurationProperties());
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.getExtension().getConfiguration().put("key.serializer",
			StringSerializer.class.getName());
	producerProperties.getExtension().setMessageKeyExpression(
			spelExpressionParser.parseExpression("headers.key"));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	String uniqueBindingId = UUID.randomUUID().toString();
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Thread.sleep(1000);
	Message<?> message = MessageBuilder.withPayload("somePayload")
			.setHeader("key", "myDynamicKey").build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	String receivedKey = new String(inbound.getHeaders()
			.get(KafkaHeaders.RECEIVED_MESSAGE_KEY, byte[].class));
	assertThat(receivedKey).isEqualTo("myDynamicKey");
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(10);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(10);
	producerProperties.setPartitionKeyExpression(new LiteralExpression("foo"));

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"foo" + uniqueBindingId + ".0", null, moduleInputChannel,
			consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(10);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testDlqWithProducerPropertiesSetAtBinderLevel()
		throws Exception {

	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();

	Map<String, String> consumerProps = new HashMap<>();
	consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
	binderConfiguration.setConsumerProperties(consumerProps);

	Map<String, String> producerProps = new HashMap<>();
	producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
	binderConfiguration.setProducerProperties(producerProps);
	Binder binder = getBinder(binderConfiguration);

	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setUseNativeEncoding(true);
	BindingProperties outputBindingProperties = createProducerBindingProperties(
			producerProperties);
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			outputBindingProperties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setUseNativeDecoding(true);
	consumerProperties.getExtension().setEnableDlq(true);

	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<MessageChannel> producerBinding = binder.bindProducer("foo.bar",
			moduleOutputChannel, outputBindingProperties.getProducer());
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo.bar",
			"tdwcapsabl", moduleInputChannel, consumerProperties);

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();

	FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler();
	moduleInputChannel.subscribe(handler);

	// Consumer for the DLQ destination
	QueueChannel dlqChannel = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> dlqConsumerProperties = createConsumerProperties();
	dlqConsumerProperties.setMaxAttempts(1);

	Binding<MessageChannel> dlqConsumerBinding = binder.bindConsumer(
			"error.foo.bar." + "tdwcapsabl", null, dlqChannel,
			dlqConsumerProperties);
	binderBindUnbindLatency();

	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload("foo").build();

	moduleOutputChannel.send(message);

	Message<?> receivedMessage = dlqChannel.receive(5000);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getPayload()).isEqualTo("foo");

	binderBindUnbindLatency();

	dlqConsumerBinding.unbind();

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@Override
@SuppressWarnings("unchecked")
public void testSendAndReceive() throws Exception {
	Binder binder = getBinder();
	BindingProperties outputBindingProperties = createProducerBindingProperties(
			createProducerProperties());
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			outputBindingProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<MessageChannel> producerBinding = binder.bindProducer("foo.bar",
			moduleOutputChannel, outputBindingProperties.getProducer());
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo.bar",
			"testSendAndReceive", moduleInputChannel, consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload("foo".getBytes(StandardCharsets.UTF_8))
			.setHeader(MessageHeaders.CONTENT_TYPE,
					MimeTypeUtils.APPLICATION_OCTET_STREAM)
			.build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	CountDownLatch latch = new CountDownLatch(1);
	AtomicReference<Message<byte[]>> inboundMessageRef = new AtomicReference<>();
	moduleInputChannel.subscribe(message1 -> {
		try {
			inboundMessageRef.set((Message<byte[]>) message1);
		}
		finally {
			latch.countDown();
		}
	});
	Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message");

	assertThat(inboundMessageRef.get()).isNotNull();
	assertThat(
			new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8))
					.isEqualTo("foo");
	assertThat(inboundMessageRef.get().getHeaders().get(MessageHeaders.CONTENT_TYPE))
			.isEqualTo(MimeTypeUtils.APPLICATION_OCTET_STREAM);

	Map<String, TopicInformation> topicsInUse = ((KafkaTestBinder) binder)
			.getCoreBinder().getTopicsInUse();
	assertThat(topicsInUse.keySet()).contains("foo.bar");
	TopicInformation topic = topicsInUse.get("foo.bar");
	assertThat(topic.isConsumerTopic()).isTrue();
	assertThat(topic.getConsumerGroup()).isEqualTo("testSendAndReceive");

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testResume() throws Exception {
	Binding<MessageChannel> producerBinding = null;
	Binding<MessageChannel> consumerBinding = null;

	try {
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
		Binder binder = getBinder(configurationProperties);

		BindingProperties producerBindingProperties = createProducerBindingProperties(
				createProducerProperties());
		DirectChannel output = createBindableChannel("output",
				producerBindingProperties);

		DirectChannel input1 = createBindableChannel("input",
				createConsumerBindingProperties(createConsumerProperties()));

		String testTopicName = UUID.randomUUID().toString();
		producerBinding = binder.bindProducer(testTopicName, output,
				producerBindingProperties.getProducer());
		ExtendedConsumerProperties<KafkaConsumerProperties> firstConsumerProperties = createConsumerProperties();
		consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1,
				firstConsumerProperties);
		CountDownLatch latch = new CountDownLatch(1);
		AtomicReference<Message<byte[]>> inboundMessageRef1 = new AtomicReference<>();
		MessageHandler messageHandler = message1 -> {
			try {
				inboundMessageRef1.set((Message<byte[]>) message1);
			}
			finally {
				latch.countDown();
			}
		};
		input1.subscribe(messageHandler);
		String testPayload1 = "foo1-" + UUID.randomUUID().toString();
		output.send(new GenericMessage<>(testPayload1));
		Assert.isTrue(latch.await(15, TimeUnit.SECONDS), "Failed to receive message");

		assertThat(inboundMessageRef1.get()).isNotNull();
		assertThat(inboundMessageRef1.get().getPayload()).isNotNull();
		input1.unsubscribe(messageHandler);
		CountDownLatch latch1 = new CountDownLatch(1);
		AtomicReference<Message<byte[]>> inboundMessageRef2 = new AtomicReference<>();
		MessageHandler messageHandler1 = message1 -> {
			try {
				inboundMessageRef2.set((Message<byte[]>) message1);
			}
			finally {
				latch1.countDown();
			}
		};
		input1.subscribe(messageHandler1);
		String testPayload2 = "foo2-" + UUID.randomUUID().toString();
		output.send(new GenericMessage<>(testPayload2.getBytes()));
		Assert.isTrue(latch1.await(15, TimeUnit.SECONDS),
				"Failed to receive message");
		assertThat(inboundMessageRef2.get()).isNotNull();
		assertThat(inboundMessageRef2.get().getPayload()).isNotNull();
		consumerBinding.unbind();

		Thread.sleep(2000);
		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
		consumerBinding = binder.bindConsumer(testTopicName, "startOffsets", input1,
				consumerProperties);
		input1.unsubscribe(messageHandler1);
		CountDownLatch latch2 = new CountDownLatch(1);
		AtomicReference<Message<byte[]>> inboundMessageRef3 = new AtomicReference<>();
		MessageHandler messageHandler2 = message1 -> {
			try {
				inboundMessageRef3.set((Message<byte[]>) message1);
			}
			finally {
				latch2.countDown();
			}
		};
		input1.subscribe(messageHandler2);
		String testPayload3 = "foo3-" + UUID.randomUUID().toString();
		output.send(new GenericMessage<>(testPayload3.getBytes()));
		Assert.isTrue(latch2.await(15, TimeUnit.SECONDS),
				"Failed to receive message");
		assertThat(inboundMessageRef3.get()).isNotNull();
		assertThat(new String(inboundMessageRef3.get().getPayload(),
				StandardCharsets.UTF_8)).isEqualTo(testPayload3);
	}
	finally {
		if (consumerBinding != null) {
			consumerBinding.unbind();
		}
		if (producerBinding != null) {
			producerBinding.unbind();
		}
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testAutoCreateTopicsDisabledOnBinderStillWorksAsLongAsBrokerCreatesTopic()
		throws Exception {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();
	configurationProperties.setAutoCreateTopics(false);
	Binder binder = getBinder(configurationProperties);
	BindingProperties producerBindingProperties = createProducerBindingProperties(
			createProducerProperties());
	DirectChannel output = createBindableChannel("output", producerBindingProperties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	DirectChannel input = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	String testTopicName = "createdByBroker-" + System.currentTimeMillis();

	Binding<MessageChannel> producerBinding = binder.bindProducer(testTopicName,
			output, producerBindingProperties.getProducer());

	String testPayload = "foo1-" + UUID.randomUUID().toString();
	output.send(new GenericMessage<>(testPayload));

	Binding<MessageChannel> consumerBinding = binder.bindConsumer(testTopicName,
			"test", input, consumerProperties);
	CountDownLatch latch = new CountDownLatch(1);
	AtomicReference<Message<byte[]>> inboundMessageRef = new AtomicReference<>();
	input.subscribe(message1 -> {
		try {
			inboundMessageRef.set((Message<byte[]>) message1);
		}
		finally {
			latch.countDown();
		}
	});
	Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message");

	assertThat(inboundMessageRef.get()).isNotNull();
	assertThat(
			new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8))
					.isEqualTo(testPayload);

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@Override
@SuppressWarnings("unchecked")
public void testSendAndReceiveMultipleTopics() throws Exception {
	Binder binder = getBinder();

	DirectChannel moduleOutputChannel1 = createBindableChannel("output1",
			createProducerBindingProperties(createProducerProperties()));
	DirectChannel moduleOutputChannel2 = createBindableChannel("output2",
			createProducerBindingProperties(createProducerProperties()));

	QueueChannel moduleInputChannel = new QueueChannel();

	ExtendedProducerProperties<KafkaProducerProperties> producer1Props = createProducerProperties();
	producer1Props.getExtension().setUseTopicHeader(true);

	Binding<MessageChannel> producerBinding1 = binder.bindProducer("foo.x",
			moduleOutputChannel1, producer1Props);
	Binding<MessageChannel> producerBinding2 = binder.bindProducer("foo.y",
			moduleOutputChannel2, createProducerProperties());

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	Binding<MessageChannel> consumerBinding1 = binder.bindConsumer("foo.x", "test1",
			moduleInputChannel, consumerProperties);
	Binding<MessageChannel> consumerBinding2 = binder.bindConsumer("foo.y", "test2",
			moduleInputChannel, consumerProperties);

	String testPayload1 = "foo1";
	Message<?> message1 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload1.getBytes()).build();
	String testPayload2 = "foo2";
	Message<?> message2 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload2.getBytes()).build();
	String testPayload3 = "foo3";
	Message<?> message3 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload3.getBytes())
			.setHeader(KafkaHeaders.TOPIC, "foo.y")
			.build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel1.send(message1);
	moduleOutputChannel2.send(message2);
	moduleOutputChannel1.send(message3);

	Message<?>[] messages = new Message[3];
	messages[0] = receive(moduleInputChannel);
	messages[1] = receive(moduleInputChannel);
	messages[2] = receive(moduleInputChannel);

	assertThat(messages[0]).isNotNull();
	assertThat(messages[1]).isNotNull();
	assertThat(messages[1]).isNotNull();
	assertThat(messages).extracting("payload").containsExactlyInAnyOrder(
			testPayload1.getBytes(), testPayload2.getBytes(), testPayload3.getBytes());
	Arrays.asList(messages).forEach(message -> {
		if (new String((byte[]) message.getPayload()).equals("foo1")) {
			assertThat(message.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC)).isEqualTo("foo.x");
		}
		else {
			assertThat(message.getHeaders().get(KafkaHeaders.RECEIVED_TOPIC)).isEqualTo("foo.y");
		}
	});

	producerBinding1.unbind();
	producerBinding2.unbind();

	consumerBinding1.unbind();
	consumerBinding2.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff() throws Exception {
	Binder binder = getBinder();

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(createProducerProperties()));
	QueueChannel moduleInputChannel = new QueueChannel();

	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff",
			moduleOutputChannel, createProducerProperties());

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().setAutoCommitOffset(false);

	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"testManualAckSucceedsWhenAutoCommitOffsetIsTurnedOff", "test",
			moduleInputChannel, consumerProperties);

	String testPayload1 = "foo" + UUID.randomUUID().toString();
	Message<?> message1 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload1.getBytes()).build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message1);

	Message<?> receivedMessage = receive(moduleInputChannel);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT))
			.isNotNull();
	Acknowledgment acknowledgment = receivedMessage.getHeaders()
			.get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class);
	try {
		acknowledgment.acknowledge();
	}
	catch (Exception e) {
		fail("Acknowledge must not throw an exception");
	}
	finally {
		producerBinding.unbind();
		consumerBinding.unbind();
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder()
		throws Exception {
	Binder binder = getBinder();

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(createProducerProperties()));
	QueueChannel moduleInputChannel = new QueueChannel();

	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			moduleOutputChannel, createProducerProperties());

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			"test", moduleInputChannel, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.BATCH);

	String testPayload1 = "foo" + UUID.randomUUID().toString();
	Message<?> message1 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload1.getBytes()).build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message1);

	Message<?> receivedMessage = receive(moduleInputChannel);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT))
			.isNull();

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@Override
@SuppressWarnings("unchecked")
public void testTwoRequiredGroups() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	String testDestination = "testDestination"
			+ UUID.randomUUID().toString().replace("-", "");

	producerProperties.setRequiredGroups("test1", "test2");
	Binding<MessageChannel> producerBinding = binder.bindProducer(testDestination,
			output, producerProperties);

	String testPayload = "foo-" + UUID.randomUUID().toString();
	output.send(new GenericMessage<>(testPayload.getBytes()));

	QueueChannel inbound1 = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	consumerProperties.getExtension().setAckEachRecord(true);
	Binding<MessageChannel> consumerBinding1 = binder.bindConsumer(testDestination,
			"test1", inbound1, consumerProperties);
	QueueChannel inbound2 = new QueueChannel();
	Binding<MessageChannel> consumerBinding2 = binder.bindConsumer(testDestination,
			"test2", inbound2, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding2, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.RECORD);

	Message<?> receivedMessage1 = receive(inbound1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String((byte[]) receivedMessage1.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);
	Message<?> receivedMessage2 = receive(inbound2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String((byte[]) receivedMessage2.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);

	consumerBinding1.unbind();
	consumerBinding2.unbind();
	producerBinding.unbind();
}
 

@Test
@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testPartitionedModuleJava() throws Exception {
	Binder binder = getBinder();

	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(4);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0J");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.0", "test",
			input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1J");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.0", "test",
			input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2J");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.0", "test",
			input2, consumerProperties);
	consumerProperties.setInstanceIndex(3);
	QueueChannel input3 = new QueueChannel();
	input3.setBeanName("test.input3J");
	Binding<MessageChannel> input3Binding = binder.bindConsumer("partJ.0", "test",
			input3, consumerProperties);

	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	this.applicationContext.registerBean("pkExtractor",
			PartitionTestSupport.class, () -> new PartitionTestSupport());
	this.applicationContext.registerBean("pkSelector",
			PartitionTestSupport.class, () -> new PartitionTestSupport());
	producerProperties.setPartitionKeyExtractorName("pkExtractor");
	producerProperties.setPartitionSelectorName("pkSelector");
	producerProperties.setPartitionCount(3); // overridden to 8 on the actual topic
	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.0", output,
			producerProperties);
	if (usesExplicitRouting()) {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("partJ.0", "test")
						+ "-' + headers['partition']");
	}

	output.send(new GenericMessage<>(2));
	output.send(new GenericMessage<>(1));
	output.send(new GenericMessage<>(0));
	output.send(new GenericMessage<>(3));

	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();
	Message<?> receive3 = receive(input3);
	assertThat(receive3).isNotNull();
	ObjectMapper om = new ObjectMapper();

	assertThat(om.readValue((byte[]) receive0.getPayload(), Integer.class))
			.isEqualTo(0);
	assertThat(om.readValue((byte[]) receive1.getPayload(), Integer.class))
			.isEqualTo(1);
	assertThat(om.readValue((byte[]) receive2.getPayload(), Integer.class))
			.isEqualTo(2);
	assertThat(om.readValue((byte[]) receive3.getPayload(), Integer.class))
			.isEqualTo(3);

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	input3Binding.unbind();
	outputBinding.unbind();
}
 

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testSendAndReceiveWithRawMode() throws Exception {
	Binder binder = getBinder();

	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setHeaderMode(HeaderMode.none);
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setHeaderMode(HeaderMode.none);
	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer("raw.0",
			moduleOutputChannel, producerProperties);

	Binding<MessageChannel> consumerBinding = binder.bindConsumer("raw.0", "test",
			moduleInputChannel, consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload("testSendAndReceiveWithRawMode".getBytes()).build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);

	CountDownLatch latch = new CountDownLatch(1);
	AtomicReference<Message<byte[]>> inboundMessageRef = new AtomicReference<>();
	moduleInputChannel.subscribe(message1 -> {
		try {
			inboundMessageRef.set((Message<byte[]>) message1);
		}
		finally {
			latch.countDown();
		}
	});
	Assert.isTrue(latch.await(5, TimeUnit.SECONDS), "Failed to receive message");

	assertThat(inboundMessageRef.get()).isNotNull();
	assertThat(
			new String(inboundMessageRef.get().getPayload(), StandardCharsets.UTF_8))
					.isEqualTo("testSendAndReceiveWithRawMode");
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testDlqWithNativeDecodingOnConsumerButMissingSerializerOnDlqProducer()
		throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	// Native serialization for producer
	producerProperties.setUseNativeEncoding(true);
	Map<String, String> producerConfig = new HashMap<>();
	producerConfig.put("value.serializer",
			"org.apache.kafka.common.serialization.StringSerializer");
	producerProperties.getExtension().setConfiguration(producerConfig);
	BindingProperties outputBindingProperties = createProducerBindingProperties(
			producerProperties);
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			outputBindingProperties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	// Native Deserialization for consumer
	consumerProperties.setUseNativeDecoding(true);

	Map<String, String> consumerConfig = new HashMap<>();
	consumerConfig.put("value.deserializer",
			"org.apache.kafka.common.serialization.StringDeserializer");

	// No Dlq producer properties set on the consumer with a native serializer.
	// This should cause an error for DLQ sending.

	consumerProperties.getExtension().setConfiguration(consumerConfig);
	consumerProperties.getExtension().setEnableDlq(true);

	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<MessageChannel> producerBinding = binder.bindProducer("foo.bar",
			moduleOutputChannel, outputBindingProperties.getProducer());
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo.bar",
			"testDlqWithNativeEncoding-2", moduleInputChannel, consumerProperties);

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();

	FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler();
	moduleInputChannel.subscribe(handler);

	// Consumer for the DLQ destination
	QueueChannel dlqChannel = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> dlqConsumerProperties = createConsumerProperties();
	dlqConsumerProperties.setMaxAttempts(1);

	Binding<MessageChannel> dlqConsumerBinding = binder.bindConsumer(
			"error.foo.bar." + "testDlqWithNativeEncoding-2", null, dlqChannel,
			dlqConsumerProperties);
	binderBindUnbindLatency();

	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload("foo").build();

	moduleOutputChannel.send(message);

	Message<?> receivedMessage = dlqChannel.receive(5000);
	// Ensure that we didn't receive anything on DLQ because of serializer config
	// missing on dlq producer while native Decoding is enabled.
	assertThat(receivedMessage).isNull();

	binderBindUnbindLatency();

	dlqConsumerBinding.unbind();

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
@Ignore
public void testDlqWithNativeSerializationEnabledOnDlqProducer() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	// Native serialization for producer
	producerProperties.setUseNativeEncoding(true);
	Map<String, String> producerConfig = new HashMap<>();
	producerConfig.put("value.serializer",
			"org.apache.kafka.common.serialization.StringSerializer");
	producerProperties.getExtension().setConfiguration(producerConfig);

	BindingProperties outputBindingProperties = createProducerBindingProperties(
			producerProperties);

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			outputBindingProperties);
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	// Native Deserialization for consumer
	consumerProperties.setUseNativeDecoding(true);
	Map<String, String> consumerConfig = new HashMap<>();
	consumerConfig.put("value.deserializer",
			"org.apache.kafka.common.serialization.StringDeserializer");
	consumerProperties.getExtension().setConfiguration(consumerConfig);

	// Setting dlq producer properties on the consumer
	consumerProperties.getExtension()
			.setDlqProducerProperties(producerProperties.getExtension());
	consumerProperties.getExtension().setEnableDlq(true);

	DirectChannel moduleInputChannel = createBindableChannel("input",
			createConsumerBindingProperties(consumerProperties));

	Binding<MessageChannel> producerBinding = binder.bindProducer("foo.bar",
			moduleOutputChannel, outputBindingProperties.getProducer());
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo.bar",
			"testDlqWithNativeEncoding-1", moduleInputChannel, consumerProperties);

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();

	FailingInvocationCountingMessageHandler handler = new FailingInvocationCountingMessageHandler();
	moduleInputChannel.subscribe(handler);

	// Consumer for the DLQ destination
	QueueChannel dlqChannel = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> dlqConsumerProperties = createConsumerProperties();
	dlqConsumerProperties.setMaxAttempts(1);

	Binding<MessageChannel> dlqConsumerBinding = binder.bindConsumer(
			"error.foo.bar." + "testDlqWithNativeEncoding-1", null, dlqChannel,
			dlqConsumerProperties);
	binderBindUnbindLatency();

	Message<?> message = org.springframework.integration.support.MessageBuilder
			.withPayload("foo").build();

	moduleOutputChannel.send(message);

	Message<?> receivedMessage = receive(dlqChannel, 5);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getPayload()).isEqualTo("foo".getBytes());
	assertThat(handler.getInvocationCount())
			.isEqualTo(consumerProperties.getMaxAttempts());
	assertThat(receivedMessage.getHeaders()
			.get(KafkaMessageChannelBinder.X_ORIGINAL_TOPIC))
					.isEqualTo("foo.bar".getBytes(StandardCharsets.UTF_8));
	assertThat(new String((byte[]) receivedMessage.getHeaders()
			.get(KafkaMessageChannelBinder.X_EXCEPTION_MESSAGE))).startsWith(
					"Dispatcher failed to deliver Message; nested exception is java.lang.RuntimeException: fail");
	assertThat(receivedMessage.getHeaders()
			.get(KafkaMessageChannelBinder.X_EXCEPTION_STACKTRACE)).isNotNull();
	binderBindUnbindLatency();

	dlqConsumerBinding.unbind();

	producerBinding.unbind();
	consumerBinding.unbind();
}