org.testcontainers.containers.KafkaContainer Java Examples

public Map<String, String> start() {
    String bootstrapServers = "localhost:9092";
    if (!skipKafkaContainer) {
        kafka = new KafkaContainer();
        kafka.addEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1");
        kafka.addEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1");
        kafka.start();

        bootstrapServers = kafka.getBootstrapServers();

        createTopics(bootstrapServers);
    }
    return Collections.singletonMap(
            CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
            bootstrapServers
    );
}
 

@Override
public Map<String, String> start() {
    LOGGER.info(TestcontainersConfiguration.getInstance().toString());

    try {
        container = new KafkaContainer(CONFLUENT_PLATFORM_VERSION)
                .withEmbeddedZookeeper()
                .waitingFor(Wait.forListeningPort());

        container.start();

        return Collections.singletonMap("camel.component.kafka.brokers", container.getBootstrapServers());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@BeforeClass
public static void setup() {
    // confluent versions 5.3.x correspond Kafka versions 2.3.x -
    // https://docs.confluent.io/current/installation/versions-interoperability.html#cp-and-apache-ak-compatibility
    kafka = new KafkaContainer("5.3.0");
    kafka.start();
    kafkaPort = kafka.getMappedPort(KafkaContainer.KAFKA_PORT);
    bootstrapServers = kafka.getBootstrapServers();
    consumerThread = new Consumer();
    consumerThread.start();
    replyConsumer = createKafkaConsumer();
    replyConsumer.subscribe(Collections.singletonList(REPLY_TOPIC));
    producer = new KafkaProducer<>(
        ImmutableMap.of(
            ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers,
            ProducerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString(),
            // This should guarantee that records are batched, as long as they are sent within the configured duration
            ProducerConfig.LINGER_MS_CONFIG, 50
        ),
        new StringSerializer(),
        new StringSerializer()
    );
}
 

@BeforeClass
public static void setup() {
    reporter.disableDestinationAddressCheck();

    // confluent versions 3.2.x correspond Kafka versions 0.10.2.2 -
    // https://docs.confluent.io/current/installation/versions-interoperability.html#cp-and-apache-ak-compatibility
    kafka = new KafkaContainer("3.2.2");
    kafka.start();
    bootstrapServers = kafka.getBootstrapServers();
    consumerThread = new Consumer();
    consumerThread.start();
    replyConsumer = createKafkaConsumer();
    replyConsumer.subscribe(Collections.singletonList(REPLY_TOPIC));
    producer = new KafkaProducer<>(
        ImmutableMap.of(
            ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers,
            ProducerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString(),
            // This should guarantee that records are batched, as long as they are sent within the configured duration
            ProducerConfig.LINGER_MS_CONFIG, 50
        ),
        new StringSerializer(),
        new StringSerializer()
    );
}
 

@BeforeClass
public static void setup() {
    // confluent versions 5.3.0 correspond Kafka versions 2.3.0 -
    // https://docs.confluent.io/current/installation/versions-interoperability.html#cp-and-apache-ak-compatibility
    kafka = new KafkaContainer("5.3.0");
    kafka.start();
    kafkaPort = kafka.getMappedPort(KafkaContainer.KAFKA_PORT);
    bootstrapServers = kafka.getBootstrapServers();
    consumerThread = new Consumer();
    consumerThread.start();
    replyConsumer = createKafkaConsumer();
    replyConsumer.subscribe(Collections.singletonList(REPLY_TOPIC));
    producer = new KafkaProducer<>(
        ImmutableMap.of(
            ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers,
            ProducerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString(),
            // This should guarantee that records are batched, as long as they are sent within the configured duration
            ProducerConfig.LINGER_MS_CONFIG, 50
        ),
        new StringSerializer(),
        new StringSerializer()
    );
}
 

@Override
protected KafkaContainer createSinkService(PulsarCluster cluster) {
    return new KafkaContainer()
            .withEmbeddedZookeeper()
            .withNetworkAliases(containerName)
            .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd
                .withName(containerName)
                .withHostName(cluster.getClusterName() + "-" + containerName));
}
 

@Bean(destroyMethod = "stop")
public KafkaContainer kafkaContainer() {
    KafkaContainer container = new KafkaContainer();
    container.withEnv("KAFKA_DELETE_TOPIC_ENABLE", "true");
    container.withEnv("KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS ", "500");
    container.start();
    return container;
}
 

@Bean
public AdminClient kafkaAdminClient(KafkaContainer kafkaContainer) {
    Map<String, Object> adminProperties = new HashMap<>();
    adminProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());

    return AdminClient.create(adminProperties);
}
 

@Bean
public ProducerFactory<?, ?> kafkaProducerFactory(KafkaContainer kafkaContainer) {
    Map<String, Object> producerProperties = new HashMap<>();
    producerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

    return new DefaultKafkaProducerFactory<>(producerProperties);
}
 

@Bean
public ConsumerFactory<?, ?> kafkaConsumerFactory(KafkaContainer kafkaContainer) {
    Map<String, Object> consumerProperties = new HashMap<>();
    consumerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());
    consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    consumerProperties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 500);
    consumerProperties.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 400);

    return new DefaultKafkaConsumerFactory<>(consumerProperties);
}
 

@Override
public Map<String, String> start() {
    kafka = new KafkaContainer();
    kafka.start();
    System.setProperty("kafka.bootstrap.servers", kafka.getBootstrapServers());
    return Collections.emptyMap();
}
 

@Override
public Map<String, String> start() {
    KafkaContainer KAFKA = new KafkaContainer();
    KAFKA.start();
    System.setProperty("kafka.bootstrap.servers", KAFKA.getBootstrapServers());
    return Collections.emptyMap();
}
 

private KafkaConsumer<byte[], byte[]> getConsumerBytes(KafkaContainer kafkaContainer) {
    return new KafkaConsumer<>(
        ImmutableMap.of(
            ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092",
            ConsumerConfig.GROUP_ID_CONFIG, "tc-" + UUID.randomUUID(),
            ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"),
        new ByteArrayDeserializer(),
        new ByteArrayDeserializer());
}
 

public void start() {
    LOGGER.info("Starting kafka container");
    kafkaContainer = new KafkaContainer();
    kafkaContainer.addEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1");
    kafkaContainer.addEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1");
    kafkaContainer.start();
}
 

SchemaRegistryContainer withKafka(KafkaContainer kafka) {
    return withKafka(kafka.getNetwork(), kafka.getNetworkAliases().get(0) + ":9092");
}
 

public LiiklusContainer withKafka(KafkaContainer kafkaContainer) {
    return withKafka(kafkaContainer.getNetwork(), kafkaContainer.getNetworkAliases().get(0) + ":9092");
}
 

public TestingKafka()
{
    container = new KafkaContainer("5.4.1");
}
 

@BeforeClass
public static void setup() {
  container = new KafkaContainer("5.3.1");
  container.start();
}
 

@BeforeEach
public void startContainers() {
    String apicurioVersion = System.getProperty("project.version");
    assertNotNull(apicurioVersion);

    Path converterDistro = Paths.get(System.getProperty("user.dir"), "..", "distro", "connect-converter",
            "target", "apicurio-kafka-connect-converter-" + apicurioVersion + "-converter.tar.gz");

    if (Files.notExists(converterDistro)) {
        LOGGER.info("Connecter distribution {}", converterDistro.toString());
        throw new IllegalStateException("Kafka connect converter distribution is not present");
    }

    ImageFromDockerfile apicurioDebeziumImage = new ImageFromDockerfile()
            .withFileFromPath("converter-distro.tar.gz", converterDistro)
            .withDockerfileFromBuilder(builder -> builder
                    .from("debezium/connect:1.1.1.Final")
                    .env("KAFKA_CONNECT_DEBEZIUM_DIR", "$KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres")
                    .copy("converter-distro.tar.gz", "$KAFKA_CONNECT_DEBEZIUM_DIR/apicurio-kafka-connect-converter.tar.gz")
                    .run("cd $KAFKA_CONNECT_DEBEZIUM_DIR && tar -xvf apicurio-kafka-connect-converter.tar.gz")
                    .build());

    if (!TestUtils.isExternalRegistry()) {
        Testcontainers.exposeHostPorts(8081);
    }

    Testcontainers.exposeHostPorts(9092);
    kafka = new KafkaContainer();
    kafka.addEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1");
    kafka.addEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1");
    kafka.addExposedPorts(9092);
    kafka.withCreateContainerCmdModifier(cmd -> {
        cmd
                .withHostName("localhost")
                .withPortBindings(new PortBinding(Ports.Binding.bindPort(9092), new ExposedPort(9092)));
    });
    kafka.start();

    Network network = Network.newNetwork();

    postgres = new PostgreSQLContainer<>("debezium/postgres:11")
          .withNetwork(network)
          .withNetworkAliases("postgres");
    postgres.start();

    debeziumContainer = new DebeziumContainer("dummy-version");
    debeziumContainer.setImage(apicurioDebeziumImage);
    debeziumContainer.withNetwork(network)
          .withEnv("BOOTSTRAP_SERVERS", "host.testcontainers.internal:9092")
          .withLogConsumer(new Slf4jLogConsumer(LOGGER));
    debeziumContainer.setWaitStrategy(
            Wait.forHttp("/connectors")
            .forPort(8083)
            .forStatusCode(200)
            .withReadTimeout(Duration.ofSeconds(3))
            .withStartupTimeout(Duration.ofSeconds(300)));
    debeziumContainer.start();

}
 

@Override
public void setServiceContainer(KafkaContainer container) {
    this.kafkaContainer = container;
}