Java Code Examples for io.vertx.kafka.client.consumer.KafkaConsumer#create()
The following examples show how to use
io.vertx.kafka.client.consumer.KafkaConsumer#create() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: VertxKafkaClientExamples.java From vertx-kafka-client with Apache License 2.0 | 6 votes |
|
public void exampleUsingVertxDeserializers2(Vertx vertx) {
Map<String, String> config = new HashMap<>();
config.put("bootstrap.servers", "localhost:9092");
config.put("group.id", "my_group");
config.put("auto.offset.reset", "earliest");
config.put("enable.auto.commit", "false");
// Creating a consumer able to deserialize buffers
KafkaConsumer<Buffer, Buffer> bufferConsumer = KafkaConsumer.create(vertx, config, Buffer.class, Buffer.class);
// Creating a consumer able to deserialize json objects
KafkaConsumer<JsonObject, JsonObject> jsonObjectConsumer = KafkaConsumer.create(vertx, config, JsonObject.class, JsonObject.class);
// Creating a consumer able to deserialize json arrays
KafkaConsumer<JsonArray, JsonArray> jsonArrayConsumer = KafkaConsumer.create(vertx, config, JsonArray.class, JsonArray.class);
}
Example 2
| Source File: ConsumerTestBase.java From vertx-kafka-client with Apache License 2.0 | 6 votes |
|
@Test
public void testPollTimeout(TestContext ctx) throws Exception {
Async async = ctx.async();
String topicName = "testPollTimeout";
Properties config = kafkaCluster.useTo().getConsumerProperties(topicName, topicName, OffsetResetStrategy.EARLIEST);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
io.vertx.kafka.client.common.TopicPartition topicPartition = new io.vertx.kafka.client.common.TopicPartition(topicName, 0);
KafkaConsumer<Object, Object> consumerWithCustomTimeout = KafkaConsumer.create(vertx, config);
int pollingTimeout = 1500;
// Set the polling timeout to 1500 ms (default is 1000)
consumerWithCustomTimeout.pollTimeout(Duration.ofMillis(pollingTimeout));
// Subscribe to the empty topic (we want the poll() call to timeout!)
consumerWithCustomTimeout.subscribe(topicName, subscribeRes -> {
consumerWithCustomTimeout.handler(rec -> {}); // Consumer will now immediately poll once
long beforeSeek = System.currentTimeMillis();
consumerWithCustomTimeout.seekToBeginning(topicPartition, seekRes -> {
long durationWShortTimeout = System.currentTimeMillis() - beforeSeek;
ctx.assertTrue(durationWShortTimeout >= pollingTimeout, "Operation must take at least as long as the polling timeout");
consumerWithCustomTimeout.close();
async.countDown();
});
});
}
Example 3
| Source File: EarliestTest.java From vertx-kafka-client with Apache License 2.0 | 6 votes |
|
@Override
public void start(Promise<Void> startFuture) throws Exception {
Map<String, String> config = new HashMap<>();
config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
config.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
KafkaConsumer consumer = KafkaConsumer.create(vertx, config);
consumer.handler(r -> {
System.out.println(r);
});
consumer.subscribe("my-topic");
startFuture.complete();
}
Example 4
| Source File: SinkBridgeEndpoint.java From strimzi-kafka-bridge with Apache License 2.0 | 6 votes |
|
/**
* Kafka consumer initialization. It should be the first call for preparing the Kafka consumer.
*/
protected void initConsumer(boolean shouldAttachBatchHandler, Properties config) {
// create a consumer
KafkaConfig kafkaConfig = this.bridgeConfig.getKafkaConfig();
Properties props = new Properties();
props.putAll(kafkaConfig.getConfig());
props.putAll(kafkaConfig.getConsumerConfig().getConfig());
props.put(ConsumerConfig.GROUP_ID_CONFIG, this.groupId);
if (this.bridgeConfig.getTracing() != null) {
props.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingConsumerInterceptor.class.getName());
}
if (config != null)
props.putAll(config);
this.consumer = KafkaConsumer.create(this.vertx, props, keyDeserializer, valueDeserializer);
if (shouldAttachBatchHandler)
this.consumer.batchHandler(this::handleKafkaBatch);
}
Example 5
| Source File: VertxKafkaClientExamples.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
/**
* Example about Kafka consumer and producer creation
* @param vertx Vert.x instance
*/
public void exampleCreateConsumer(Vertx vertx) {
// creating the consumer using map config
Map<String, String> config = new HashMap<>();
config.put("bootstrap.servers", "localhost:9092");
config.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
config.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
config.put("group.id", "my_group");
config.put("auto.offset.reset", "earliest");
config.put("enable.auto.commit", "false");
// use consumer for interacting with Apache Kafka
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
}
Example 6
| Source File: ConsumerTestBase.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
@Test
public void testConsumerBatchHandler(TestContext ctx) throws Exception {
String topicName = "testConsumerBatchHandler";
String consumerId = topicName;
Async batch1 = ctx.async();
AtomicInteger index = new AtomicInteger();
int numMessages = 500;
kafkaCluster.useTo().produceStrings(numMessages, batch1::complete, () ->
new ProducerRecord<>(topicName, 0, "key-" + index.get(), "value-" + index.getAndIncrement()));
batch1.awaitSuccess(10000);
Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
KafkaConsumer<Object, Object> wrappedConsumer = KafkaConsumer.create(vertx, config);
wrappedConsumer.exceptionHandler(ctx::fail);
AtomicInteger count = new AtomicInteger(numMessages);
Async batchHandler = ctx.async();
batchHandler.handler(ar -> wrappedConsumer.close());
wrappedConsumer.batchHandler(records -> {
ctx.assertEquals(numMessages, records.size());
for (int i = 0; i < records.size(); i++) {
KafkaConsumerRecord<Object, Object> record = records.recordAt(i);
int dec = count.decrementAndGet();
if (dec >= 0) {
ctx.assertEquals("key-" + (numMessages - dec - 1), record.key());
} else {
ctx.assertEquals("key-" + (-1 - dec), record.key());
}
}
batchHandler.complete();
});
wrappedConsumer.handler(rec -> {});
wrappedConsumer.subscribe(Collections.singleton(topicName));
}
Example 7
| Source File: ConsumerTestBase.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
@Test
public void testNotCommitted(TestContext ctx) throws Exception {
String topicName = "testNotCommitted";
String consumerId = topicName;
kafkaCluster.createTopic(topicName, 1, 1);
Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
Async done = ctx.async();
KafkaConsumer<Object, Object> consumer = KafkaConsumer.create(vertx, config);
consumer.handler(rec -> {});
consumer.partitionsAssignedHandler(partitions -> {
for (io.vertx.kafka.client.common.TopicPartition partition : partitions) {
consumer.committed(partition, ar -> {
if (ar.succeeded()) {
ctx.assertNull(ar.result());
} else {
ctx.fail(ar.cause());
}
});
}
done.complete();
});
consumer.subscribe(Collections.singleton(topicName));
}
Example 8
| Source File: ConsumerTestBase.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
@Test
public void testConsumeWithPoll(TestContext ctx) {
final String topicName = "testConsumeWithPoll";
final String consumerId = topicName;
Async batch = ctx.async();
int numMessages = 1000;
kafkaCluster.useTo().produceStrings(numMessages, batch::complete, () ->
new ProducerRecord<>(topicName, "value")
);
batch.awaitSuccess(20000);
Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
Async done = ctx.async();
AtomicInteger count = new AtomicInteger(numMessages);
consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {
if (subscribeResult.succeeded()) {
vertx.setPeriodic(1000, t -> {
consumer.poll(Duration.ofMillis(100), pollResult -> {
if (pollResult.succeeded()) {
if (count.updateAndGet(o -> count.get() - pollResult.result().size()) == 0) {
vertx.cancelTimer(t);
done.complete();
}
} else {
ctx.fail();
}
});
});
} else {
ctx.fail();
}
});
}
Example 9
| Source File: ConsumerTestBase.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
@Test
public void testConsumeWithPollNoMessages(TestContext ctx) {
final String topicName = "testConsumeWithPollNoMessages";
final String consumerId = topicName;
Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
Async done = ctx.async();
AtomicInteger count = new AtomicInteger(5);
consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {
if (subscribeResult.succeeded()) {
vertx.setPeriodic(1000, t -> {
consumer.poll(Duration.ofMillis(100), pollResult -> {
if (pollResult.succeeded()) {
if (pollResult.result().size() > 0) {
ctx.fail();
} else {
if (count.decrementAndGet() == 0) {
vertx.cancelTimer(t);
done.complete();
}
}
} else {
ctx.fail();
}
});
});
} else {
ctx.fail();
}
});
}
Example 10
| Source File: VertxKafkaClientExamples.java From vertx-kafka-client with Apache License 2.0 | 5 votes |
|
/**
* Example about Kafka consumer and producer creation
* @param vertx Vert.x instance
*/
public void exampleCreateConsumerJava(Vertx vertx) {
// creating the consumer using properties config
Properties config = new Properties();
config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
config.put(ConsumerConfig.GROUP_ID_CONFIG, "my_group");
config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
// use consumer for interacting with Apache Kafka
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
}
Example 11
| Source File: Consumer.java From strimzi-kafka-operator with Apache License 2.0 | 5 votes |
|
Consumer(KafkaClientProperties properties, CompletableFuture<Integer> resultPromise, IntPredicate msgCntPredicate, String topic, String clientName) {
super(resultPromise, msgCntPredicate);
this.properties = properties;
this.topic = topic;
this.clientName = clientName;
this.vertx = Vertx.vertx();
this.consumer = KafkaConsumer.create(vertx, properties.getProperties());
}
Example 12
| Source File: AvroConsumerVertx.java From df_data_service with Apache License 2.0 | 4 votes |
|
@Override
public void start() throws Exception {
System.out.println("Test");
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, "group1");
props.put("schema.registry.url", "http://localhost:8002");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
String topic = "test_stock";
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, props);
ArrayList<JsonObject> responseList = new ArrayList<JsonObject>();
// consumer.handler(record -> {// TODO handler does not work
// System.out.println("Processing value=" + record.record().value() +
// ",partition=" + record.record().partition() + ",offset=" + record.record().offset());
// responseList.add(new JsonObject()
// .put("offset", record.record().offset())
// .put("value", record.record().value().toString()));
// if(responseList.size() >= 10 ) {
// consumer.pause();
// consumer.commit();
// consumer.close();
// }
// });
//
// // Subscribe to a single topic
// consumer.subscribe(topic, ar -> {
// if (ar.succeeded()) {
// System.out.println("topic " + topic + " is subscribed");
// } else {
// System.out.println("Could not subscribe " + ar.cause().getMessage());
// }
// });
consumer.partitionsFor(topic, ar -> {
if (ar.succeeded()) {
for (PartitionInfo partitionInfo : ar.result()) {
System.out.println(partitionInfo);
}
}
});
}
Example 13
| Source File: DFDataProcessor.java From df_data_service with Apache License 2.0 | 4 votes |
|
/**
* Poll all available information from specific topic
* @param routingContext
*
* @api {get} /avroconsumer 7.List all df tasks using specific topic
* @apiVersion 0.1.1
* @apiName poolAllFromTopic
* @apiGroup All
* @apiPermission none
* @apiDescription This is where consume data from specific topic in one pool.
* @apiSuccess {JsonObject[]} topic Consumer from the topic.
* @apiSampleRequest http://localhost:8080/api/df/avroconsumer
*/
private void pollAllFromTopic(RoutingContext routingContext) {
final String topic = routingContext.request().getParam("id");
if (topic == null) {
routingContext.response()
.setStatusCode(ConstantApp.STATUS_CODE_BAD_REQUEST)
.end(DFAPIMessage.getResponseMessage(9000));
LOG.error(DFAPIMessage.getResponseMessage(9000, "TOPIC_IS_NULL"));
} else {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_server_host_and_port);
props.put(ConsumerConfig.GROUP_ID_CONFIG, ConstantApp.DF_CONNECT_KAFKA_CONSUMER_GROUP_ID);
props.put(ConstantApp.SCHEMA_URI_KEY, "http://" + schema_registry_host_and_port);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, props);
ArrayList<JsonObject> responseList = new ArrayList<JsonObject>();
consumer.handler(record -> {
//LOG.debug("Processing value=" + record.record().value() + ",offset=" + record.record().offset());
responseList.add(new JsonObject()
.put("id", record.record().offset())
.put("value", new JsonObject(record.record().value().toString()))
.put("valueString", Json.encodePrettily(new JsonObject(record.record().value().toString())))
);
if(responseList.size() >= ConstantApp.AVRO_CONSUMER_BATCH_SIE ) {
HelpFunc.responseCorsHandleAddOn(routingContext.response())
.putHeader("X-Total-Count", responseList.size() + "")
.end(Json.encodePrettily(responseList));
consumer.pause();
consumer.commit();
consumer.close();
}
});
consumer.exceptionHandler(e -> {
LOG.error(DFAPIMessage.logResponseMessage(9031, topic + "-" + e.getMessage()));
});
// Subscribe to a single topic
consumer.subscribe(topic, ar -> {
if (ar.succeeded()) {
LOG.info(DFAPIMessage.logResponseMessage(1027, "topic = " + topic));
} else {
LOG.error(DFAPIMessage.logResponseMessage(9030, topic + "-" + ar.cause().getMessage()));
}
});
}
}
Example 14
| Source File: DFDataProcessor.java From df_data_service with Apache License 2.0 | 4 votes |
|
/**
* Describe topic with topic specified
*
* @api {get} /s2p/:taskId 6. Get partition information for the specific subject/topic
* @apiVersion 0.1.1
* @apiName getAllTopicPartitions
* @apiGroup All
* @apiPermission none
* @apiDescription This is where we get partition information for the subject/topic.
* @apiParam {String} topic topic name.
* @apiSuccess {JsonObject[]} info. partition info.
* @apiSampleRequest http://localhost:8080/api/df/s2p/:taskId
*/
private void getAllTopicPartitions(RoutingContext routingContext) {
final String topic = routingContext.request().getParam("id");
if (topic == null) {
routingContext.response()
.setStatusCode(ConstantApp.STATUS_CODE_BAD_REQUEST)
.end(DFAPIMessage.getResponseMessage(9000));
LOG.error(DFAPIMessage.getResponseMessage(9000, topic));
} else {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_server_host_and_port);
props.put(ConsumerConfig.GROUP_ID_CONFIG, ConstantApp.DF_CONNECT_KAFKA_CONSUMER_GROUP_ID);
props.put(ConstantApp.SCHEMA_URI_KEY, "http://" + schema_registry_host_and_port);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, props);
ArrayList<JsonObject> responseList = new ArrayList<JsonObject>();
// Subscribe to a single topic
consumer.partitionsFor(topic, ar -> {
if (ar.succeeded()) {
for (PartitionInfo partitionInfo : ar.result()) {
responseList.add(new JsonObject()
.put("id", partitionInfo.getTopic())
.put("partitionNumber", partitionInfo.getPartition())
.put("leader", partitionInfo.getLeader().getIdString())
.put("replicas", StringUtils.join(partitionInfo.getReplicas(), ','))
.put("insyncReplicas", StringUtils.join(partitionInfo.getInSyncReplicas(), ','))
);
HelpFunc.responseCorsHandleAddOn(routingContext.response())
.putHeader("X-Total-Count", responseList.size() + "")
.end(Json.encodePrettily(responseList));
consumer.close();
}
} else {
LOG.error(DFAPIMessage.logResponseMessage(9030, topic + "-" +
ar.cause().getMessage()));
}
});
consumer.exceptionHandler(e -> {
LOG.error(DFAPIMessage.logResponseMessage(9031, topic + "-" + e.getMessage()));
});
}
}
Example 15
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Test
void sendMultipleMessages(VertxTestContext context) {
String topic = "sendMultipleMessages";
kafkaCluster.createTopic(topic, 1, 1);
String value = "message-value";
int numMessages = MULTIPLE_MAX_MESSAGE;
JsonArray records = new JsonArray();
for (int i = 0; i < numMessages; i++) {
JsonObject json = new JsonObject();
json.put("value", value + "-" + i);
records.add(json);
}
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
.sendJsonObject(root, ar -> {
context.verify(() -> assertThat(ar.succeeded(), is(true)));
HttpResponse<JsonObject> response = ar.result();
assertThat(response.statusCode(), is(HttpResponseStatus.OK.code()));
JsonObject bridgeResponse = response.body();
JsonArray offsets = bridgeResponse.getJsonArray("offsets");
assertThat(offsets.size(), is(numMessages));
for (int i = 0; i < numMessages; i++) {
JsonObject metadata = offsets.getJsonObject(i);
assertThat(metadata.getInteger("partition"), is(0));
assertThat(metadata.getLong("offset"), notNullValue());
}
});
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
new StringDeserializer(), new KafkaJsonDeserializer<String>(String.class));
this.count = 0;
consumer.handler(record -> {
context.verify(() -> {
assertThat(record.value(), is(value + "-" + this.count++));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), notNullValue());
assertThat(record.offset(), notNullValue());
assertThat(record.key(), nullValue());
});
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
if (this.count == numMessages) {
consumer.close();
context.completeNow();
}
});
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
Example 16
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Test
void sendPeriodicMessage(VertxTestContext context) {
String topic = "sendPeriodicMessage";
kafkaCluster.createTopic(topic, 1, 1);
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
new KafkaJsonDeserializer<>(String.class), new KafkaJsonDeserializer<>(String.class));
this.count = 0;
vertx.setPeriodic(HttpBridgeTestBase.PERIODIC_DELAY, timerId -> {
if (this.count < HttpBridgeTestBase.PERIODIC_MAX_MESSAGE) {
JsonArray records = new JsonArray();
JsonObject json = new JsonObject();
json.put("value", "Periodic message [" + this.count + "]");
json.put("key", "key-" + this.count);
records.add(json);
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
.sendJsonObject(root, ar -> { });
this.count++;
} else {
vertx.cancelTimer(timerId);
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
});
consumer.batchHandler(records -> {
context.verify(() -> {
assertThat(records.size(), is(this.count));
for (int i = 0; i < records.size(); i++) {
KafkaConsumerRecord<String, String> record = records.recordAt(i);
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
assertThat(record.value(), is("Periodic message [" + i + "]"));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), notNullValue());
assertThat(record.offset(), notNullValue());
assertThat(record.key(), is("key-" + i));
}
});
consumer.close();
context.completeNow();
});
consumer.handler(record -> { });
}
Example 17
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Disabled("Will be check in the next PR, this is just external tests for Bridge")
@DisabledIfEnvironmentVariable(named = "BRIDGE_EXTERNAL_ENV", matches = "((?i)FALSE(?-i))")
@Test
void sendBinaryMessageWithKey(VertxTestContext context) {
String topic = "sendBinaryMessageWithKey";
kafkaCluster.createTopic(topic, 2, 1);
String value = "message-value";
String key = "my-key-bin";
JsonArray records = new JsonArray();
JsonObject json = new JsonObject();
json.put("value", DatatypeConverter.printBase64Binary(value.getBytes()));
json.put("key", DatatypeConverter.printBase64Binary(key.getBytes()));
records.add(json);
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_BINARY)
.sendJsonObject(root, verifyOK(context));
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<byte[], byte[]> consumer = KafkaConsumer.create(vertx, config,
new ByteArrayDeserializer(), new ByteArrayDeserializer());
consumer.handler(record -> {
context.verify(() -> {
assertThat(new String(record.value()), is(value));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), notNullValue());
assertThat(record.offset(), is(0L));
assertThat(new String(record.key()), is(key));
});
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
consumer.close();
context.completeNow();
});
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
Example 18
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Test
void sendSimpleMessageWithKey(VertxTestContext context) {
String topic = "sendSimpleMessageWithKey";
kafkaCluster.createTopic(topic, 2, 1);
String value = "message-value";
String key = "my-key";
JsonArray records = new JsonArray();
JsonObject json = new JsonObject();
json.put("value", value);
json.put("key", key);
records.add(json);
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
.sendJsonObject(root, verifyOK(context));
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
new KafkaJsonDeserializer<>(String.class), new KafkaJsonDeserializer<>(String.class));
consumer.handler(record -> {
context.verify(() -> {
assertThat(record.value(), is(value));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), notNullValue());
assertThat(record.offset(), is(0L));
assertThat(record.key(), is(key));
});
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
consumer.close();
context.completeNow();
});
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
Example 19
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Test
void sendSimpleMessageToPartition(VertxTestContext context) {
String topic = "sendSimpleMessageToPartition";
kafkaCluster.createTopic(topic, 2, 1);
String value = "message-value";
int partition = 1;
JsonArray records = new JsonArray();
JsonObject json = new JsonObject();
json.put("value", value);
json.put("partition", partition);
records.add(json);
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
.sendJsonObject(root, verifyOK(context));
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
new StringDeserializer(), new KafkaJsonDeserializer<>(String.class));
consumer.handler(record -> {
context.verify(() -> {
assertThat(record.value(), is(value));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), is(partition));
assertThat(record.offset(), is(0L));
assertThat(record.key(), nullValue());
});
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
consumer.close();
context.completeNow();
});
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
Example 20
| Source File: ProducerTest.java From strimzi-kafka-bridge with Apache License 2.0 | 4 votes |
|
@Test
void sendSimpleMessage(VertxTestContext context) {
String topic = "sendSimpleMessage";
kafkaCluster.createTopic(topic, 1, 1);
String value = "message-value";
JsonArray records = new JsonArray();
JsonObject json = new JsonObject();
json.put("value", value);
records.add(json);
JsonObject root = new JsonObject();
root.put("records", records);
producerService()
.sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
.sendJsonObject(root, verifyOK(context));
Properties config = kafkaCluster.getConsumerProperties();
KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
new StringDeserializer(), new KafkaJsonDeserializer<>(String.class));
consumer.handler(record -> {
context.verify(() -> {
assertThat(record.value(), is(value));
assertThat(record.topic(), is(topic));
assertThat(record.partition(), is(0));
assertThat(record.offset(), is(0L));
assertThat(record.key(), nullValue());
});
LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
record.topic(), record.partition(), record.offset(), record.key(), record.value());
consumer.close();
context.completeNow();
});
consumer.subscribe(topic, done -> {
if (!done.succeeded()) {
context.failNow(done.cause());
}
});
}
