org.springframework.kafka.listener.ContainerProperties Java Examples

private void registerContainers(ConfigurableListableBeanFactory beanFactory,
		List<Contract> matchingContracts, String flowName,
		StubRunnerKafkaRouter listener) {
	// listener's container
	ConsumerFactory consumerFactory = beanFactory.getBean(ConsumerFactory.class);
	for (Contract matchingContract : matchingContracts) {
		if (matchingContract.getInput() == null) {
			continue;
		}
		String destination = MapConverter.getStubSideValuesForNonBody(
				matchingContract.getInput().getMessageFrom()).toString();
		ContainerProperties containerProperties = new ContainerProperties(
				destination);
		KafkaMessageListenerContainer container = listenerContainer(consumerFactory,
				containerProperties, listener);
		String containerName = flowName + ".container";
		Object initializedContainer = beanFactory.initializeBean(container,
				containerName);
		beanFactory.registerSingleton(containerName, initializedContainer);
		if (log.isDebugEnabled()) {
			log.debug(
					"Initialized kafka message container with name [" + containerName
							+ "] listening to destination [" + destination + "]");
		}
	}
}
 

/**
 * 消费则的监听工厂
 * @return
 */
@Bean
public KafkaListenerContainerFactory userRegisterSuccKafkaListenerContainerFactory(){
    ConcurrentKafkaListenerContainerFactory conFactory = new ConcurrentKafkaListenerContainerFactory<>();
    conFactory.setConsumerFactory(kafKaRegisterSuccConsumerFactory());
    conFactory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);//  设置消费者消费消息后的提交方式为手动提交
    return conFactory;
}
 

@Test
public void test() throws InterruptedException {
    log.info("Start auto");
    ContainerProperties containerProps = new ContainerProperties("topic1", "topic2");
    final CountDownLatch latch = new CountDownLatch(4);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        log.info("received: " + message);
        latch.countDown();
    });
    KafkaMessageListenerContainer<Integer, String> container = createContainer(containerProps);
    container.setBeanName("testAuto");
    container.start();
    Thread.sleep(1000); // wait a bit for the container to start
    KafkaTemplate<Integer, String> template = createTemplate();
    template.setDefaultTopic("zptest");
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    container.stop();
    log.info("Stop auto");
}
 

@Bean({"eventsKafkaListenerContainerFactory", "kafkaListenerContainerFactory"})
public ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> eventsKafkaListenerContainerFactory(
        EventMessageConverter eventMessageConverter, ConsumerFactory<String, PublishedEventWrapper> consumerFactory) {

    ConcurrentKafkaListenerContainerFactory<String, PublishedEventWrapper> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory);
    factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getEventConcurrency());
    factory.setMessageConverter(eventMessageConverter);
    factory.getContainerProperties().setPollTimeout(3000);
    ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
    scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getEventSchedulerPoolSize());
    scheduler.setBeanName("EventsFactory-Scheduler");
    scheduler.initialize();

    factory.getContainerProperties().setScheduler(scheduler);
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
    return factory;
}
 

@Bean
public KafkaMessageListenerContainer<String, String> domainEventListenerContainer(KafkaDomainEventListener domainEventListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(eventTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(domainEventListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 

@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(ConsumerFactory<Object, Object> kafkaConsumerFactory) {
    ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(kafkaConsumerFactory);

    ContainerProperties containerProperties = factory.getContainerProperties();
    containerProperties.setMissingTopicsFatal(true);
    return factory;
}
 

private KafkaMessageListenerContainer listenerContainer(
		ConsumerFactory consumerFactory, ContainerProperties containerProperties,
		GenericMessageListener listener) {
	KafkaMessageListenerContainer container = new KafkaMessageListenerContainer(
			consumerFactory, containerProperties);
	container.setupMessageListener(listener);
	return container;
}
 

@Bean("ackContainerFactory")
public ConcurrentKafkaListenerContainerFactory<String, String> ackContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
    factory.setConcurrency(KafkaConsts.DEFAULT_PARTITION_NUM);
    return factory;
}
 

@Bean(name = "kafkaListenerContainerFactory")
public ConcurrentKafkaListenerContainerFactory<?, ?> kafkaListenerContainerFactory(
    ConcurrentKafkaListenerContainerFactoryConfigurer configurer,
    ConsumerFactory<Object, Object> kafkaConsumerFactory) {
  ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
  ContainerProperties containerProperties = factory.getContainerProperties();
  factory.setRecordFilterStrategy(locMessageFilterStrategy());
  factory.setErrorHandler(new LocKafkaConsumerErrorHandler());
  factory.setMessageConverter(recordMessageConverter());
  configurer.configure(factory, kafkaConsumerFactory);
  return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);

    return factory;
}
 

@Bean("operationsKafkaListenerContainerFactory")
    public ConcurrentKafkaListenerContainerFactory<String, Operation> operationsKafkaListenerContainerFactory(
            ConsumerFactory<String, Operation> consumerFactory) {
        ConcurrentKafkaListenerContainerFactory<String, Operation> factory
                = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory);
        RetryTemplate retryTemplate = new RetryTemplate();
        factory.setRetryTemplate(retryTemplate);

        factory.setConcurrency(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
        scheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        scheduler.setBeanName("OperationsFactory-Scheduler");
        scheduler.initialize();
        factory.getContainerProperties().setScheduler(scheduler);
        ThreadPoolTaskScheduler consumerScheduler = new ThreadPoolTaskScheduler();
        consumerScheduler.setPoolSize(eventApisConfiguration.getEventBus().getConsumer().getOperationSchedulerPoolSize());
        consumerScheduler.setBeanName("OperationsFactory-ConsumerScheduler");
        consumerScheduler.initialize();

        factory.getContainerProperties().setPollTimeout(3000L);
        factory.getContainerProperties().setAckOnError(false);
        factory.getContainerProperties().setConsumerTaskExecutor(consumerScheduler);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.RECORD);
        /**
         * This is Fix for Spring Kafka versions which does not have ConsumerAwareErrorHandler handler till 2.0
         * When Listener faces with error, it retries snapshot operation
         * See https://github.com/kloiasoft/eventapis/issues/44
         */
        factory.getContainerProperties().setTransactionManager(new EmptyTransactionManager());
//        factory.getContainerProperties().setTransactionManager(platformTransactionManager);
        return factory;
    }
 

private KafkaMessageListenerContainer<Integer, String> createContainer(
    ContainerProperties containerProps) {
    Map<String, Object> props = consumerProps();
    DefaultKafkaConsumerFactory<Integer, String> cf =
        new DefaultKafkaConsumerFactory<>(props);
    KafkaMessageListenerContainer<Integer, String> container =
        new KafkaMessageListenerContainer<>(cf, containerProps);
    return container;
}
 

@Bean
public KafkaMessageListenerContainer publishableExceptionListenerContainer(KafkaPublishableExceptionListener publishableExceptionListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.EXCEPTION_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(publishableExceptionListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 

@Bean
public KafkaMessageListenerContainer applicationMessageListenerContainer(KafkaApplicationMessageListener applicationMessageListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.APPLICATION_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(applicationMessageListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 

@Bean
public KafkaMessageListenerContainer domainEventListenerContainer(KafkaDomainEventListener domainEventListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(QueueProperties.EVENT_TOPIC);
    properties.setGroupId(QueueProperties.DEFAULT_PRODUCER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(domainEventListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 

@Bean
public KafkaMessageListenerContainer kafkaMessageListenerContainer(KafkaCommandListener commandListener, RetryTemplate retryTemplate) {
    ContainerProperties properties = new ContainerProperties(COMMAND_TOPIC);
    properties.setGroupId(DEFAULT_CONSUMER_GROUP);
    RetryingMessageListenerAdapter listenerAdapter = new RetryingMessageListenerAdapter(commandListener, retryTemplate);
    properties.setMessageListener(listenerAdapter);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory(), properties);
}
 

@Bean
public KafkaMessageListenerContainer<String, String> publishableExceptionListenerContainer(KafkaPublishableExceptionListener publishableExceptionListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(exceptionTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(publishableExceptionListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 

@Bean
public KafkaMessageListenerContainer<String, String> applicationMessageListenerContainer(KafkaApplicationMessageListener applicationMessageListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(applicationTopic);
    properties.setGroupId(Constants.DEFAULT_PRODUCER_GROUP);
    properties.setMessageListener(applicationMessageListener);
    properties.setMissingTopicsFatal(false);
    properties.setAckMode(ContainerProperties.AckMode.MANUAL);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 

@Bean
public KafkaMessageListenerContainer<String, String> commandListenerContainer(KafkaCommandListener commandListener, ConsumerFactory<String, String> consumerFactory) {
    ContainerProperties properties = new ContainerProperties(commandTopic);
    properties.setGroupId(Constants.DEFAULT_CONSUMER_GROUP);
    properties.setMessageListener(commandListener);
    properties.setMissingTopicsFatal(false);
    return new KafkaMessageListenerContainer<>(consumerFactory, properties);
}
 

/**
 * @return replyContainer
 */
public KafkaMessageListenerContainer<Object, Object> replyContainer() {
    ContainerProperties containerProperties = new ContainerProperties(properties.getProducer().getReplyTopic());
    return new KafkaMessageListenerContainer<>(consumerFactory, containerProperties);
}
 

@Test
@SuppressWarnings("unchecked")
public void testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder()
		throws Exception {
	Binder binder = getBinder();

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(createProducerProperties()));
	QueueChannel moduleInputChannel = new QueueChannel();

	Binding<MessageChannel> producerBinding = binder.bindProducer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			moduleOutputChannel, createProducerProperties());

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

	Binding<MessageChannel> consumerBinding = binder.bindConsumer(
			"testManualAckIsNotPossibleWhenAutoCommitOffsetIsEnabledOnTheBinder",
			"test", moduleInputChannel, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.BATCH);

	String testPayload1 = "foo" + UUID.randomUUID().toString();
	Message<?> message1 = org.springframework.integration.support.MessageBuilder
			.withPayload(testPayload1.getBytes()).build();

	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message1);

	Message<?> receivedMessage = receive(moduleInputChannel);
	assertThat(receivedMessage).isNotNull();
	assertThat(receivedMessage.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT))
			.isNull();

	producerBinding.unbind();
	consumerBinding.unbind();
}
 

@Test
@Override
@SuppressWarnings("unchecked")
public void testTwoRequiredGroups() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();

	DirectChannel output = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	String testDestination = "testDestination"
			+ UUID.randomUUID().toString().replace("-", "");

	producerProperties.setRequiredGroups("test1", "test2");
	Binding<MessageChannel> producerBinding = binder.bindProducer(testDestination,
			output, producerProperties);

	String testPayload = "foo-" + UUID.randomUUID().toString();
	output.send(new GenericMessage<>(testPayload.getBytes()));

	QueueChannel inbound1 = new QueueChannel();
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	consumerProperties.getExtension().setAckEachRecord(true);
	Binding<MessageChannel> consumerBinding1 = binder.bindConsumer(testDestination,
			"test1", inbound1, consumerProperties);
	QueueChannel inbound2 = new QueueChannel();
	Binding<MessageChannel> consumerBinding2 = binder.bindConsumer(testDestination,
			"test2", inbound2, consumerProperties);

	AbstractMessageListenerContainer<?, ?> container = TestUtils.getPropertyValue(
			consumerBinding2, "lifecycle.messageListenerContainer",
			AbstractMessageListenerContainer.class);
	assertThat(container.getContainerProperties().getAckMode())
			.isEqualTo(ContainerProperties.AckMode.RECORD);

	Message<?> receivedMessage1 = receive(inbound1);
	assertThat(receivedMessage1).isNotNull();
	assertThat(new String((byte[]) receivedMessage1.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);
	Message<?> receivedMessage2 = receive(inbound2);
	assertThat(receivedMessage2).isNotNull();
	assertThat(new String((byte[]) receivedMessage2.getPayload(),
			StandardCharsets.UTF_8)).isEqualTo(testPayload);

	consumerBinding1.unbind();
	consumerBinding2.unbind();
	producerBinding.unbind();
}
 

@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledSucceedsIfTopicPartitionedCorrectly()
		throws Throwable {
	Binding<?> binding = null;
	try {
		KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

		String testTopicName = "existing" + System.currentTimeMillis();
		invokeCreateTopic(testTopicName, 6, 1);
		configurationProperties.setAutoAddPartitions(false);
		Binder binder = getBinder(configurationProperties);
		ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();

		DirectChannel input = createBindableChannel("input",
				createConsumerBindingProperties(consumerProperties));

		// this consumer must consume from partition 2
		consumerProperties.setInstanceCount(3);
		consumerProperties.setInstanceIndex(2);
		consumerProperties.getExtension().setAutoRebalanceEnabled(false);

		binding = binder.bindConsumer(testTopicName, "test-x", input,
				consumerProperties);

		ContainerProperties containerProps = TestUtils.getPropertyValue(
				binding,
				"lifecycle.messageListenerContainer.containerProperties",
				ContainerProperties.class);
		TopicPartitionOffset[] listenedPartitions = containerProps.getTopicPartitionsToAssign();
		assertThat(listenedPartitions).hasSize(2);
		assertThat(listenedPartitions).contains(
				new TopicPartitionOffset(testTopicName, 2),
				new TopicPartitionOffset(testTopicName, 5));
		int partitions = invokePartitionSize(testTopicName);
		assertThat(partitions).isEqualTo(6);
	}
	finally {
		if (binding != null) {
			binding.unbind();
		}
	}
}
 

@Before
public void setUp() throws Exception {
  // set up the Kafka consumer properties
  Map<String, Object> consumerProperties =
      KafkaTestUtils.consumerProps("sender", "false",
          embeddedKafka.getEmbeddedKafka());

  // create a Kafka consumer factory
  DefaultKafkaConsumerFactory<String, String> consumerFactory =
      new DefaultKafkaConsumerFactory<String, String>(
          consumerProperties);

  // set the topic that needs to be consumed
  ContainerProperties containerProperties =
      new ContainerProperties(SENDER_TOPIC);

  // create a Kafka MessageListenerContainer
  container = new KafkaMessageListenerContainer<>(consumerFactory,
      containerProperties);

  // create a thread safe queue to store the received message
  records = new LinkedBlockingQueue<>();

  // setup a Kafka message listener
  container
      .setupMessageListener(new MessageListener<String, String>() {
        @Override
        public void onMessage(
            ConsumerRecord<String, String> record) {
          LOGGER.debug("test-listener received message='{}'",
              record.toString());
          records.add(record);
        }
      });

  // start the container and underlying message listener
  container.start();

  // wait until the container has the required number of assigned partitions
  ContainerTestUtils.waitForAssignment(container,
      embeddedKafka.getEmbeddedKafka().getPartitionsPerTopic());
}