org.springframework.kafka.test.EmbeddedKafkaBroker Java Examples

@BeforeAll
public static void init(EmbeddedKafkaBroker embeddedKafka) {
    Map<String, Object> props = new HashMap<>();

    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, embeddedKafka.getBrokersAsString());
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
    setEventGenerator(new KafkaTemplateInventoryUpdateEventGenerator(props, INPUT_TOPIC));

    Map<String, Object> consumerProps = KafkaTestUtils.consumerProps(GROUP_NAME, "true", embeddedKafka);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
    consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "test");
    consumerProps.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 1000);
    consumerProps.put(JsonDeserializer.TRUSTED_PACKAGES, KafkaStreamsInventoryCountTests.class.getPackage().getName());
    consumerProps.put(JsonDeserializer.KEY_DEFAULT_TYPE, ProductKey.class);
    consumerProps.put(JsonDeserializer.VALUE_DEFAULT_TYPE, InventoryCountEvent.class);
    consumerProps.put(JsonDeserializer.USE_TYPE_INFO_HEADERS, "false");
    cf = new DefaultKafkaConsumerFactory<>(consumerProps);


    /*
     * Disabling caching makes the test run faster, and more consistent behavior with the TopologyTestDriver.
     * More messages are produced on the output topic.
     */
    context = new SpringApplicationBuilder(KafkaStreamsInventoryCountApplication.class)
            .properties(
                    "spring.cloud.stream.kafka.streams.binder.brokers=" + embeddedKafka.getBrokersAsString(),
                    "spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000",
                    "spring.cloud.stream.kafka.streams.binder.configuration.cache.max.bytes.buffering=0")
            .run();
}
 

@Override
public Map<String, Consumer> initialize(EmbeddedKafkaBroker broker,
		KafkaProperties kafkaProperties) {
	Map<String, Consumer> map = new HashMap<>();
	for (String topic : broker.getTopics()) {
		map.put(topic, prepareListener(broker, topic, kafkaProperties));
	}
	return map;
}
 

KafkaStubMessages(KafkaTemplate kafkaTemplate, EmbeddedKafkaBroker broker,
		KafkaProperties kafkaProperties, KafkaStubMessagesInitializer initializer) {
	this.kafkaTemplate = kafkaTemplate;
	Map<String, Consumer> topicToConsumer = initializer.initialize(broker,
			kafkaProperties);
	this.receiver = new Receiver(topicToConsumer);
}
 

@Bean
@ConditionalOnMissingBean
MessageVerifier<Message<?>> contractVerifierKafkaMessageExchange(
		KafkaTemplate kafkaTemplate, EmbeddedKafkaBroker broker,
		KafkaProperties kafkaProperties, KafkaStubMessagesInitializer initializer) {
	return new KafkaStubMessages(kafkaTemplate, broker, kafkaProperties, initializer);
}
 

@Bean
public ProducerFactory<String, String> producerFactory(final EmbeddedKafkaBroker embeddedKafkaBroker) {
    final Map<String, Object> configs = producerProps(embeddedKafkaBroker);
    configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    return new DefaultKafkaProducerFactory<>(configs);
}
 

/**
 * Build a minimal {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 *
 * @param kafkaBroker       the {@link EmbeddedKafkaBroker} used in the test case
 * @param valueDeserializer a {@link Class} defining the type of value deserializer to be used
 * @return a minimal {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 */
public static Map<String, Object> minimal(EmbeddedKafkaBroker kafkaBroker, Class valueDeserializer) {
    Map<String, Object> config = new HashMap<>();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker.getBrokersAsString());
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer);
    config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    return config;
}
 

private Consumer prepareListener(EmbeddedKafkaBroker broker, String destination,
		KafkaProperties kafkaProperties) {
	Map<String, Object> consumerProperties = KafkaTestUtils.consumerProps(
			kafkaProperties.getConsumer().getGroupId(), "false", broker);
	consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
	DefaultKafkaConsumerFactory<String, String> consumerFactory = new DefaultKafkaConsumerFactory<>(
			consumerProperties);
	Consumer<String, String> consumer = consumerFactory.createConsumer();
	broker.consumeFromAnEmbeddedTopic(consumer, destination);
	if (log.isDebugEnabled()) {
		log.debug("Prepared consumer for destination [" + destination + "]");
	}
	return consumer;
}
 

protected void setUp(KafkaProducerFactory producerFactory, KafkaConsumerFactory consumerFactory,
    KafkaProperties properties, EmbeddedKafkaBroker broker) {

    this.producerFactory = producerFactory;
    this.consumerFactory = consumerFactory;
    properties.setConsumer(addBootstrapServersToConfig(properties.getConsumer(), broker.getBrokersAsString()));
    properties.setProducer(addBootstrapServersToConfig(properties.getProducer(), broker.getBrokersAsString()));
}
 

@Bean
public EmbeddedKafkaBroker embeddedKafkaBroker() {
	return new EmbeddedKafkaBroker(1);
}
 

/**
 * Minimal configuration required for creating a {@link KafkaProducer}.
 * <ul>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker     the {@link EmbeddedKafkaBroker} being used for testing
 * @param valueSerializer the serializer for <code>value</code> that implements {@link Serializer}.
 * @return the configuration.
 */
public static Map<String, Object> minimal(EmbeddedKafkaBroker kafkaBroker, Class valueSerializer) {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker.getBrokersAsString());
    configs.put(ProducerConfig.RETRIES_CONFIG, 0);
    configs.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
    configs.put(ProducerConfig.LINGER_MS_CONFIG, 1);
    configs.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
    configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer);
    return configs;
}
 

/**
 * Factory for generating transactional {@link KafkaProducer} with:
 * <ul>
 * <li><code>confirmationMode</code> - {@link ConfirmationMode#TRANSACTIONAL}.</li>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker           the {@link EmbeddedKafkaBroker} being used for testing
 * @param transactionalIdPrefix prefix for generating <code>transactional.id</code>.
 * @param valueSerializer       The serializer for <code>value</code> that implements {@link Serializer}.
 * @return the producer factory.
 */
public static <V> ProducerFactory<String, V> transactionalProducerFactory(EmbeddedKafkaBroker kafkaBroker,
                                                                          String transactionalIdPrefix,
                                                                          Class valueSerializer) {
    return DefaultProducerFactory.<String, V>builder()
            .closeTimeout(100, ChronoUnit.MILLIS)
            .configuration(minimalTransactional(kafkaBroker, valueSerializer))
            .transactionalIdPrefix(transactionalIdPrefix)
            .build();
}
 

/**
 * Build a minimal, transactional, {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 *
 * @param kafkaBroker       the {@link EmbeddedKafkaBroker} used in the test case
 * @param valueDeserializer a {@link Class} defining the type of value deserializer to be used
 * @return a minimal, transactional, {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 */
@SuppressWarnings("WeakerAccess")
public static Map<String, Object> minimalTransactional(EmbeddedKafkaBroker kafkaBroker, Class valueDeserializer) {
    Map<String, Object> configs = minimal(kafkaBroker, valueDeserializer);
    configs.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
    return configs;
}
 

/**
 * Factory for generating transactional {@link KafkaProducer} with:
 * <ul>
 * <li><code>confirmationMode</code> - {@link ConfirmationMode#TRANSACTIONAL}.</li>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * <li><code>value.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker           the {@link EmbeddedKafkaBroker} being used for testing
 * @param transactionalIdPrefix prefix for generating <code>transactional.id</code>.
 * @return the producer factory.
 */
public static ProducerFactory<String, String> transactionalProducerFactory(EmbeddedKafkaBroker kafkaBroker,
                                                                           String transactionalIdPrefix) {
    return DefaultProducerFactory.<String, String>builder()
            .closeTimeout(100, ChronoUnit.MILLIS)
            .configuration(minimalTransactional(kafkaBroker))
            .transactionalIdPrefix(transactionalIdPrefix)
            .build();
}
 

/**
 * Factory for generating {@link KafkaProducer} with:
 * <ul>
 * <li><code>confirmationMode</code> - {@link ConfirmationMode#WAIT_FOR_ACK}.</li>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * <li><code>value.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker     the {@link EmbeddedKafkaBroker} being used for testing
 * @param valueSerializer The serializer for <code>value</code> that implements {@link Serializer}.
 * @return the producer factory.
 */
public static <V> ProducerFactory<String, V> ackProducerFactory(EmbeddedKafkaBroker kafkaBroker,
                                                                Class valueSerializer) {
    return DefaultProducerFactory.<String, V>builder()
            .closeTimeout(1000, ChronoUnit.MILLIS)
            .configuration(minimal(kafkaBroker, valueSerializer))
            .confirmationMode(WAIT_FOR_ACK)
            .build();
}
 

/**
 * Factory for generating {@link KafkaProducer} with:
 * <ul>
 * <li><code>confirmationMode</code> - {@link ConfirmationMode#NONE}.</li>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * <li><code>value.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker the Kafka.
 * @return the producer factory.
 */
public static ProducerFactory<String, String> producerFactory(EmbeddedKafkaBroker kafkaBroker) {
    return DefaultProducerFactory.<String, String>builder()
            .configuration(minimal(kafkaBroker))
            .closeTimeout(100, ChronoUnit.MILLIS)
            .build();
}
 

/**
 * Minimal configuration required for creating a transactional {@link KafkaProducer}.
 * <ul>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker     the {@link EmbeddedKafkaBroker} being used for testing
 * @param valueSerializer the serializer for <code>value</code> that implements {@link Serializer}.
 * @return the configuration.
 */
public static Map<String, Object> minimalTransactional(EmbeddedKafkaBroker kafkaBroker, Class valueSerializer) {
    Map<String, Object> configs = minimal(kafkaBroker, valueSerializer);
    configs.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);
    configs.put(ProducerConfig.RETRIES_CONFIG, 1);
    return configs;
}
 

/**
 * @param broker - embedded Kafka broker
 * @param kafkaProperties - kafka properties
 * @return topic to initialized consumer mapping
 */
Map<String, Consumer> initialize(EmbeddedKafkaBroker broker,
		KafkaProperties kafkaProperties);
 

/**
 * Build a minimal {@link ConsumerFactory} to be used during testing only.
 *
 * @param kafkaBroker the {@link EmbeddedKafkaBroker} used in the test case
 * @return a {@link ConsumerFactory} configured with the minimal properties based on the given {@code kafkaBroker}
 */
public static ConsumerFactory<String, String> consumerFactory(EmbeddedKafkaBroker kafkaBroker) {
    return new DefaultConsumerFactory<>(minimal(kafkaBroker));
}
 

/**
 * Build a minimal {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 *
 * @param kafkaBroker the {@link EmbeddedKafkaBroker} used in the test case
 * @return a minimal {@link org.apache.kafka.clients.consumer.Consumer} configuration {@link Map}
 */
public static Map<String, Object> minimal(EmbeddedKafkaBroker kafkaBroker) {
    return minimal(kafkaBroker, StringDeserializer.class);
}
 

/**
 * Build a minimal, transactional {@link ConsumerFactory} to be used during testing only.
 *
 * @param kafkaBroker       the {@link EmbeddedKafkaBroker} used in the test case
 * @param valueDeserializer a {@link Class} defining the type of value deserializer to be used
 * @return a {@link ConsumerFactory} configured with the minimal properties based on the given {@code kafkaBroker}
 * and {@code valueDeserializer}
 */
public static ConsumerFactory<String, Object> transactionalConsumerFactory(EmbeddedKafkaBroker kafkaBroker,
                                                                           Class valueDeserializer) {
    return new DefaultConsumerFactory<>(minimalTransactional(kafkaBroker, valueDeserializer));
}
 

/**
 * Minimal configuration required for creating a transactional {@link KafkaProducer}.
 * <ul>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * <li><code>value.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker the {@link EmbeddedKafkaBroker} being used for testing
 * @return the configuration.
 */
public static Map<String, Object> minimalTransactional(EmbeddedKafkaBroker kafkaBroker) {
    return minimalTransactional(kafkaBroker, StringSerializer.class);
}
 

/**
 * Minimal configuration required for creating a {@link KafkaProducer}.
 * <ul>
 * <li><code>key.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * <li><code>value.serializer</code> - {@link org.apache.kafka.common.serialization.StringSerializer}.</li>
 * </ul>
 *
 * @param kafkaBroker the {@link EmbeddedKafkaBroker} being used for testing
 * @return the configuration.
 */
public static Map<String, Object> minimal(EmbeddedKafkaBroker kafkaBroker) {
    return minimal(kafkaBroker, StringSerializer.class);
}