Java Code Examples for io.vertx.kafka.client.consumer.KafkaConsumer#subscribe()

public void exampleSubscribe(KafkaConsumer<String, String> consumer) {
  // register the handler for incoming messages
  consumer.handler(record -> {
    System.out.println("Processing key=" + record.key() + ",value=" + record.value() +
      ",partition=" + record.partition() + ",offset=" + record.offset());
  });

  // subscribe to several topics with list
  Set<String> topics = new HashSet<>();
  topics.add("topic1");
  topics.add("topic2");
  topics.add("topic3");
  consumer.subscribe(topics);

  // or using a Java regex
  Pattern pattern = Pattern.compile("topic\\d");
  consumer.subscribe(pattern);

  // or just subscribe to a single topic
  consumer.subscribe("a-single-topic");
}
 

@Test
public void testPollTimeout(TestContext ctx) throws Exception {
  Async async = ctx.async();
  String topicName = "testPollTimeout";
  Properties config = kafkaCluster.useTo().getConsumerProperties(topicName, topicName, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  io.vertx.kafka.client.common.TopicPartition topicPartition = new io.vertx.kafka.client.common.TopicPartition(topicName, 0);
  KafkaConsumer<Object, Object> consumerWithCustomTimeout = KafkaConsumer.create(vertx, config);

  int pollingTimeout = 1500;
  // Set the polling timeout to 1500 ms (default is 1000)
  consumerWithCustomTimeout.pollTimeout(Duration.ofMillis(pollingTimeout));
  // Subscribe to the empty topic (we want the poll() call to timeout!)
  consumerWithCustomTimeout.subscribe(topicName, subscribeRes -> {
    consumerWithCustomTimeout.handler(rec -> {}); // Consumer will now immediately poll once
    long beforeSeek = System.currentTimeMillis();
    consumerWithCustomTimeout.seekToBeginning(topicPartition, seekRes -> {
      long durationWShortTimeout = System.currentTimeMillis() - beforeSeek;
      ctx.assertTrue(durationWShortTimeout >= pollingTimeout, "Operation must take at least as long as the polling timeout");
      consumerWithCustomTimeout.close();
      async.countDown();
    });
  });
}
 

@Override
public void start(Promise<Void> startFuture) throws Exception {

  Map<String, String> config = new HashMap<>();
  config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
  config.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
  config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
  config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

  KafkaConsumer consumer = KafkaConsumer.create(vertx, config);
  consumer.handler(r -> {

    System.out.println(r);
  });
  consumer.subscribe("my-topic");

  startFuture.complete();
}
 

@Test
public void testConsumerBatchHandler(TestContext ctx) throws Exception {
  String topicName = "testConsumerBatchHandler";
  String consumerId = topicName;
  Async batch1 = ctx.async();
  AtomicInteger index = new AtomicInteger();
  int numMessages = 500;
  kafkaCluster.useTo().produceStrings(numMessages, batch1::complete, () ->
    new ProducerRecord<>(topicName, 0, "key-" + index.get(), "value-" + index.getAndIncrement()));
  batch1.awaitSuccess(10000);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  KafkaConsumer<Object, Object> wrappedConsumer = KafkaConsumer.create(vertx, config);
  wrappedConsumer.exceptionHandler(ctx::fail);
  AtomicInteger count = new AtomicInteger(numMessages);
  Async batchHandler = ctx.async();
  batchHandler.handler(ar -> wrappedConsumer.close());
  wrappedConsumer.batchHandler(records -> {
    ctx.assertEquals(numMessages, records.size());
    for (int i = 0; i < records.size(); i++) {
      KafkaConsumerRecord<Object, Object> record = records.recordAt(i);
      int dec = count.decrementAndGet();
      if (dec >= 0) {
        ctx.assertEquals("key-" + (numMessages - dec - 1), record.key());
      } else {
        ctx.assertEquals("key-" + (-1 - dec), record.key());
      }
    }
    batchHandler.complete();
  });
  wrappedConsumer.handler(rec -> {});
  wrappedConsumer.subscribe(Collections.singleton(topicName));
}
 

@Test
public void testNotCommitted(TestContext ctx) throws Exception {

  String topicName = "testNotCommitted";
  String consumerId = topicName;
  kafkaCluster.createTopic(topicName, 1, 1);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

  Async done = ctx.async();

  KafkaConsumer<Object, Object> consumer = KafkaConsumer.create(vertx, config);
  consumer.handler(rec -> {});
  consumer.partitionsAssignedHandler(partitions -> {
    for (io.vertx.kafka.client.common.TopicPartition partition : partitions) {
      consumer.committed(partition, ar -> {
        if (ar.succeeded()) {
          ctx.assertNull(ar.result());
        } else {
          ctx.fail(ar.cause());
        }
      });
    }
    done.complete();
  });

  consumer.subscribe(Collections.singleton(topicName));
}
 

@Test
public void testConsumeWithPoll(TestContext ctx) {
  final String topicName = "testConsumeWithPoll";
  final String consumerId = topicName;
  Async batch = ctx.async();
  int numMessages = 1000;
  kafkaCluster.useTo().produceStrings(numMessages, batch::complete, () ->
    new ProducerRecord<>(topicName, "value")
  );
  batch.awaitSuccess(20000);
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
  Async done = ctx.async();
  AtomicInteger count = new AtomicInteger(numMessages);

  consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {

    if (subscribeResult.succeeded()) {

      vertx.setPeriodic(1000, t -> {
        consumer.poll(Duration.ofMillis(100), pollResult -> {
          if (pollResult.succeeded()) {
            if (count.updateAndGet(o -> count.get() - pollResult.result().size()) == 0) {
              vertx.cancelTimer(t);
              done.complete();
            }
          } else {
            ctx.fail();
          }
        });
      });

    } else {
      ctx.fail();
    }
  });
}
 

@Test
public void testConsumeWithPollNoMessages(TestContext ctx) {
  final String topicName = "testConsumeWithPollNoMessages";
  final String consumerId = topicName;
  Properties config = kafkaCluster.useTo().getConsumerProperties(consumerId, consumerId, OffsetResetStrategy.EARLIEST);
  config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config);
  Async done = ctx.async();
  AtomicInteger count = new AtomicInteger(5);

  consumer.subscribe(Collections.singleton(topicName), subscribeResult -> {

    if (subscribeResult.succeeded()) {

      vertx.setPeriodic(1000, t -> {
        consumer.poll(Duration.ofMillis(100), pollResult -> {
          if (pollResult.succeeded()) {
            if (pollResult.result().size() > 0) {
              ctx.fail();
            } else {
              if (count.decrementAndGet() == 0) {
                vertx.cancelTimer(t);
                done.complete();
              }
            }
          } else {
            ctx.fail();
          }
        });
      });

    } else {
      ctx.fail();
    }
  });
}
 

@Test
void sendSimpleMessage(VertxTestContext context) {
    String topic = "sendSimpleMessage";
    kafkaCluster.createTopic(topic, 1, 1);

    String value = "message-value";

    JsonArray records = new JsonArray();
    JsonObject json = new JsonObject();
    json.put("value", value);
    records.add(json);

    JsonObject root = new JsonObject();
    root.put("records", records);

    producerService()
        .sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
        .sendJsonObject(root, verifyOK(context));

    Properties config = kafkaCluster.getConsumerProperties();

    KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
            new StringDeserializer(), new KafkaJsonDeserializer<>(String.class));
    consumer.handler(record -> {
        context.verify(() -> {
            assertThat(record.value(), is(value));
            assertThat(record.topic(), is(topic));
            assertThat(record.partition(), is(0));
            assertThat(record.offset(), is(0L));
            assertThat(record.key(), nullValue());
        });
        LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
                record.topic(), record.partition(), record.offset(), record.key(), record.value());
        consumer.close();
        context.completeNow();
    });

    consumer.subscribe(topic, done -> {
        if (!done.succeeded()) {
            context.failNow(done.cause());
        }
    });
}
 

@Test
void sendSimpleMessageToPartition(VertxTestContext context) {
    String topic = "sendSimpleMessageToPartition";
    kafkaCluster.createTopic(topic, 2, 1);

    String value = "message-value";

    int partition = 1;

    JsonArray records = new JsonArray();
    JsonObject json = new JsonObject();
    json.put("value", value);
    json.put("partition", partition);
    records.add(json);

    JsonObject root = new JsonObject();
    root.put("records", records);

    producerService()
        .sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
        .sendJsonObject(root, verifyOK(context));

    Properties config = kafkaCluster.getConsumerProperties();

    KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
            new StringDeserializer(), new KafkaJsonDeserializer<>(String.class));
    consumer.handler(record -> {
        context.verify(() -> {
            assertThat(record.value(), is(value));
            assertThat(record.topic(), is(topic));
            assertThat(record.partition(), is(partition));
            assertThat(record.offset(), is(0L));
            assertThat(record.key(), nullValue());
        });
        LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
                record.topic(), record.partition(), record.offset(), record.key(), record.value());
        consumer.close();
        context.completeNow();
    });

    consumer.subscribe(topic, done -> {
        if (!done.succeeded()) {
            context.failNow(done.cause());
        }
    });
}
 

@Test
void sendSimpleMessageWithKey(VertxTestContext context) {
    String topic = "sendSimpleMessageWithKey";
    kafkaCluster.createTopic(topic, 2, 1);

    String value = "message-value";
    String key = "my-key";

    JsonArray records = new JsonArray();
    JsonObject json = new JsonObject();
    json.put("value", value);
    json.put("key", key);
    records.add(json);

    JsonObject root = new JsonObject();
    root.put("records", records);

    producerService()
        .sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
        .sendJsonObject(root, verifyOK(context));

    Properties config = kafkaCluster.getConsumerProperties();

    KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
            new KafkaJsonDeserializer<>(String.class), new KafkaJsonDeserializer<>(String.class));
    consumer.handler(record -> {
        context.verify(() -> {
            assertThat(record.value(), is(value));
            assertThat(record.topic(), is(topic));
            assertThat(record.partition(), notNullValue());
            assertThat(record.offset(), is(0L));
            assertThat(record.key(), is(key));
        });
        LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
                record.topic(), record.partition(), record.offset(), record.key(), record.value());
        consumer.close();
        context.completeNow();
    });

    consumer.subscribe(topic, done -> {
        if (!done.succeeded()) {
            context.failNow(done.cause());
        }
    });
}
 

@Disabled("Will be check in the next PR, this is just external tests for Bridge")
@DisabledIfEnvironmentVariable(named = "BRIDGE_EXTERNAL_ENV", matches = "((?i)FALSE(?-i))")
@Test
void sendBinaryMessageWithKey(VertxTestContext context) {
    String topic = "sendBinaryMessageWithKey";
    kafkaCluster.createTopic(topic, 2, 1);

    String value = "message-value";
    String key = "my-key-bin";

    JsonArray records = new JsonArray();
    JsonObject json = new JsonObject();
    json.put("value", DatatypeConverter.printBase64Binary(value.getBytes()));
    json.put("key", DatatypeConverter.printBase64Binary(key.getBytes()));
    records.add(json);

    JsonObject root = new JsonObject();
    root.put("records", records);

    producerService()
        .sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_BINARY)
            .sendJsonObject(root, verifyOK(context));

    Properties config = kafkaCluster.getConsumerProperties();

    KafkaConsumer<byte[], byte[]> consumer = KafkaConsumer.create(vertx, config,
            new ByteArrayDeserializer(), new ByteArrayDeserializer());
    consumer.handler(record -> {
        context.verify(() -> {
            assertThat(new String(record.value()), is(value));
            assertThat(record.topic(), is(topic));
            assertThat(record.partition(), notNullValue());
            assertThat(record.offset(), is(0L));
            assertThat(new String(record.key()), is(key));
        });

        LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
                record.topic(), record.partition(), record.offset(), record.key(), record.value());
        consumer.close();
        context.completeNow();
    });

    consumer.subscribe(topic, done -> {
        if (!done.succeeded()) {
            context.failNow(done.cause());
        }
    });
}
 

@Test
void sendMultipleMessages(VertxTestContext context) {
    String topic = "sendMultipleMessages";
    kafkaCluster.createTopic(topic, 1, 1);

    String value = "message-value";

    int numMessages = MULTIPLE_MAX_MESSAGE;

    JsonArray records = new JsonArray();
    for (int i = 0; i < numMessages; i++) {
        JsonObject json = new JsonObject();
        json.put("value", value + "-" + i);
        records.add(json);
    }
    JsonObject root = new JsonObject();
    root.put("records", records);

    producerService()
        .sendRecordsRequest(topic, root, BridgeContentType.KAFKA_JSON_JSON)
        .sendJsonObject(root, ar -> {
            context.verify(() -> assertThat(ar.succeeded(), is(true)));

            HttpResponse<JsonObject> response = ar.result();
            assertThat(response.statusCode(), is(HttpResponseStatus.OK.code()));
            JsonObject bridgeResponse = response.body();

            JsonArray offsets = bridgeResponse.getJsonArray("offsets");
            assertThat(offsets.size(), is(numMessages));
            for (int i = 0; i < numMessages; i++) {
                JsonObject metadata = offsets.getJsonObject(i);
                assertThat(metadata.getInteger("partition"), is(0));
                assertThat(metadata.getLong("offset"), notNullValue());
            }
        });

    Properties config = kafkaCluster.getConsumerProperties();

    KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, config,
            new StringDeserializer(), new KafkaJsonDeserializer<String>(String.class));
    this.count = 0;
    consumer.handler(record -> {
        context.verify(() -> {
            assertThat(record.value(), is(value + "-" + this.count++));
            assertThat(record.topic(), is(topic));
            assertThat(record.partition(), notNullValue());
            assertThat(record.offset(), notNullValue());
            assertThat(record.key(), nullValue());
        });

        LOGGER.info("Message consumed topic={} partition={} offset={}, key={}, value={}",
                record.topic(), record.partition(), record.offset(), record.key(), record.value());

        if (this.count == numMessages) {
            consumer.close();
            context.completeNow();
        }
    });

    consumer.subscribe(topic, done -> {
        if (!done.succeeded()) {
            context.failNow(done.cause());
        }
    });
}
 

/**
 * Poll all available information from specific topic
 * @param routingContext
 *
 * @api {get} /avroconsumer 7.List all df tasks using specific topic
 * @apiVersion 0.1.1
 * @apiName poolAllFromTopic
 * @apiGroup All
 * @apiPermission none
 * @apiDescription This is where consume data from specific topic in one pool.
 * @apiSuccess	{JsonObject[]}	topic    Consumer from the topic.
 * @apiSampleRequest http://localhost:8080/api/df/avroconsumer
 */
private void pollAllFromTopic(RoutingContext routingContext) {

    final String topic = routingContext.request().getParam("id");
    if (topic == null) {
        routingContext.response()
                .setStatusCode(ConstantApp.STATUS_CODE_BAD_REQUEST)
                .end(DFAPIMessage.getResponseMessage(9000));
        LOG.error(DFAPIMessage.getResponseMessage(9000, "TOPIC_IS_NULL"));
    } else {
            Properties props = new Properties();
            props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_server_host_and_port);
            props.put(ConsumerConfig.GROUP_ID_CONFIG, ConstantApp.DF_CONNECT_KAFKA_CONSUMER_GROUP_ID);
            props.put(ConstantApp.SCHEMA_URI_KEY, "http://" + schema_registry_host_and_port);
            props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
            props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
            props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
            props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);

            KafkaConsumer<String, String> consumer = KafkaConsumer.create(vertx, props);
            ArrayList<JsonObject> responseList = new ArrayList<JsonObject>();

            consumer.handler(record -> {
                //LOG.debug("Processing value=" + record.record().value() + ",offset=" + record.record().offset());
                responseList.add(new JsonObject()
                        .put("id", record.record().offset())
                        .put("value", new JsonObject(record.record().value().toString()))
                        .put("valueString", Json.encodePrettily(new JsonObject(record.record().value().toString())))
                );
                if(responseList.size() >= ConstantApp.AVRO_CONSUMER_BATCH_SIE ) {
                    HelpFunc.responseCorsHandleAddOn(routingContext.response())
                            .putHeader("X-Total-Count", responseList.size() + "")
                            .end(Json.encodePrettily(responseList));
                    consumer.pause();
                    consumer.commit();
                    consumer.close();
                }
            });
            consumer.exceptionHandler(e -> {
                LOG.error(DFAPIMessage.logResponseMessage(9031, topic + "-" + e.getMessage()));
            });

            // Subscribe to a single topic
            consumer.subscribe(topic, ar -> {
                if (ar.succeeded()) {
                    LOG.info(DFAPIMessage.logResponseMessage(1027, "topic = " + topic));
                } else {
                    LOG.error(DFAPIMessage.logResponseMessage(9030, topic + "-" + ar.cause().getMessage()));
                }
            });
    }
}