org.springframework.kafka.config.KafkaListenerContainerFactory Java Examples

@Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> containerFactory = new ConcurrentKafkaListenerContainerFactory<>();
    containerFactory.setConcurrency(concurrency);

    Map<String, Object> config = Maps.newHashMap();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    // 由于课程原版实现中广告的索引数据是存在于ConcurrentHashMap中, 即每个索引服务实例的jvm中。
    // 所以当每一个索引实例监听kafka topic数据时, 需要保证每个实例都处于不同的消费者组
    // 即各个实例之间需要各不相同的groupId, 保证在部署多实例时, 每个实例都可以加载到完整的索引数据

    // 但在本实现中由于将索引数据单独提出, 存放到了Redis数据库中, 所以应该让所有实例属于同一个消费者组
    // 共同消费kafka topic下的数据, 保证索引数据不会被重复消费。

    // 综上, 若索引数据的存放如果为各个实例自身的jvm, 应该考虑加上以下代码(或自行编写其他实现)保证各实例处于不同的消费者组
    // 若索引数据存放的位置, 是所有检索实例共享的位置, 应该将以下配置取消(或直接删除本类)
    config.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
    DefaultKafkaConsumerFactory<String, String> consumerFactory = new DefaultKafkaConsumerFactory<String, String>(config);
    containerFactory.setConsumerFactory(consumerFactory);
    return containerFactory;
}
 

/**
 * 消费则的监听工厂
 * @return
 */
@Bean
public KafkaListenerContainerFactory userRegisterSuccKafkaListenerContainerFactory(){
    ConcurrentKafkaListenerContainerFactory conFactory = new ConcurrentKafkaListenerContainerFactory<>();
    conFactory.setConsumerFactory(kafKaRegisterSuccConsumerFactory());
    conFactory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL);//  设置消费者消费消息后的提交方式为手动提交
    return conFactory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
    kafkaListenerContainerFactory(ConsumerFactory<String, String> consumerFactory) {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory);
  return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 

protected KafkaListenerContainerFactory<?> resolveContainerFactory(KafkaListenerEndpoint endpoint, KafkaListenerContainerFactory<?> containerFactory) {
    if (containerFactory != null) {
        return containerFactory;
    } else if (this.containerFactory != null) {
        return this.containerFactory;
    } else if (containerFactoryBeanName != null) {
        Assert.state(beanFactory != null, "BeanFactory must be set to obtain container factory by bean name");
        // Consider changing this if live change of the factory is required...
        this.containerFactory = beanFactory.getBean(containerFactoryBeanName, KafkaListenerContainerFactory.class);
        return this.containerFactory;
    } else {
        throw new IllegalStateException("Could not resolve the " +
            KafkaListenerContainerFactory.class.getSimpleName() + " to use for [" +
            endpoint + "] no factory was given and no default is set.");
    }
}
 

@Bean(BEAN_KAFKA_BATCH_FACTORY)
@EnableKafkaCollectReceiver
@SuppressWarnings({ "unchecked", "rawtypes" })
public KafkaListenerContainerFactory<?> batchFactory(ReceiverProperties conf) {
	// Create consumer factory.
	Properties properties = conf.getKafka().getProperties();
	Map<String, Object> config = (Map) properties;
	ConsumerFactory<String, String> cf = new DefaultKafkaConsumerFactory<>(config);

	// Create concurrent consumer container factory.
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(cf);
	factory.setConcurrency(conf.getKafka().getConcurrency());
	factory.setBatchListener(true);

	// Spring kafka container properties.
	ContainerProperties containerProps = factory.getContainerProperties();
	containerProps.setPollTimeout(conf.getKafka().getPollTimeout());
	// Bulk consumption change buffer queue size.
	containerProps.setQueueDepth(conf.getKafka().getQueueDepth());
	containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);
	return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<String, String>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(concurrency);
    factory.getContainerProperties().setPollTimeout(1500);
    return factory;
}
 

/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());

  return factory;
}
 

/**
 * Register a new {@link KafkaListenerEndpoint} alongside the
 * {@link KafkaListenerContainerFactory} to use to create the underlying container.
 * <p>The {@code factory} may be {@code null} if the default factory has to be
 * used for that endpoint.
 */
protected void registerEndpoint(KafkaListenerEndpoint endpoint, KafkaListenerContainerFactory<?> factory) {
    Assert.notNull(endpoint, "Endpoint must not be null");
    Assert.hasText(endpoint.getId(), "Endpoint id must be set");

    Assert.state(this.endpointRegistry != null, "No KafkaListenerEndpointRegistry set");
    endpointRegistry.registerListenerContainer(endpoint, resolveContainerFactory(endpoint, factory), true);
}
 

/**
 * return KafkaListenerContainerFactory.<BR/>
 *
 * @return KafkaListenerContainerFactory
 */
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    log.debug("kafkaListenerContainerFactory()");
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(Settings2.HANDLER_COUNT);
    factory.getContainerProperties().setPollTimeout(Settings2.getPollTimeout());
    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(3);
    factory.getContainerProperties().setPollTimeout(3000);
    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {

    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<String, String>();
    factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(4);
    factory.getContainerProperties().setPollTimeout(4000);

    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory =
            new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);

    return factory;
}
 

@Bean(name = "kafkaListenerContainerFactory1")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory1() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
		new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory("groupB"));
	factory.setConcurrency(3);
	factory.getContainerProperties().setPollTimeout(3000);
	return factory;
}
 

@Bean(name = "kafkaListenerContainerFactory")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory =
		new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory("groupA"));
	factory.setConcurrency(3);
	factory.getContainerProperties().setPollTimeout(3000);
	return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 

@Bean(name = "kafkaListenerContainerFactoryForBatchConsumer")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactoryForBatchConsumer() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConcurrency(1);
    factory.setBatchListener(true);
    factory.setConsumerFactory(consumerFactory());
    return factory;
}
 

@Bean(name = "kafkaListenerContainerFactoryWith6Consumer")
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactoryWith3Consumer() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConcurrency(6); //3 partition -> 6 thread in parallel in a single consumer
    factory.setConsumerFactory(consumerFactory());
    return factory;
}
 

@Bean
@Primary
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());

    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>>
    kafkaListenerContainerFactory(ConsumerFactory<String, String> consumerFactory) {
  ConcurrentKafkaListenerContainerFactory<String, String> factory =
      new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory);
  return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
    factory.setConsumerFactory(consumerFactory());
    factory.getContainerProperties().setPollTimeout(1500);
    return factory;
}
 

@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
	ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
	factory.setConsumerFactory(consumerFactory());
	factory.setConcurrency(concurrency);
	factory.getContainerProperties().setPollTimeout(1500);
	factory.setBatchListener(true);
	return factory;
}
 

private KafkaListenerContainerFactory createContainerFactory(int aConcurrency) {
  ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory();
  factory.setConcurrency(aConcurrency);

  return factory;
}
 

public KafkaListenerContainerFactory<?> getContainerFactory() {
    return containerFactory;
}
 

public void setContainerFactory(KafkaListenerContainerFactory<?> containerFactory) {
    this.containerFactory = containerFactory;
}
 

public BinlogConsumer(ISender iSender, KafkaListenerContainerFactory containerFactory) {
    this.iSender = iSender;
    this.containerFactory = containerFactory;
}