io.vertx.reactivex.kafka.client.consumer.KafkaConsumer Java Examples

@Override
public Completable rxStart() {
  eventConsumer = KafkaConsumer.create(vertx, KafkaConfig.consumer("activity-service"));
  updateProducer = KafkaProducer.create(vertx, KafkaConfig.producer());
  pgPool = PgPool.pool(vertx, PgConfig.pgConnectOpts(), new PoolOptions());

  eventConsumer
    .subscribe("incoming.steps")
    .toFlowable()
    .flatMap(this::insertRecord)
    .flatMap(this::generateActivityUpdate)
    .flatMap(this::commitKafkaConsumerOffset)
    .doOnError(err -> logger.error("Woops", err))
    .retryWhen(this::retryLater)
    .subscribe();

  return Completable.complete();
}
 

@BeforeEach
void resetPgAndKafka(Vertx vertx, VertxTestContext testContext) {
  consumer = KafkaConsumer.create(vertx, KafkaConfig.consumer("activity-service-test-" + System.currentTimeMillis()));
  producer = KafkaProducer.create(vertx, KafkaConfig.producer());
  KafkaAdminClient adminClient = KafkaAdminClient.create(vertx, KafkaConfig.producer());

  PgPool pgPool = PgPool.pool(vertx, PgConfig.pgConnectOpts(), new PoolOptions());
  pgPool.query("DELETE FROM stepevent")
    .rxExecute()
    .flatMapCompletable(rs -> adminClient.rxDeleteTopics(Arrays.asList("incoming.steps", "daily.step.updates")))
    .andThen(Completable.fromAction(pgPool::close))
    .onErrorComplete()
    .subscribe(
      testContext::completeNow,
      testContext::failNow);
}
 

@Override
public Completable rxStart() {
  mailClient = MailClient.createShared(vertx, MailerConfig.config());
  webClient = WebClient.create(vertx);

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumerConfig("congrats-service"))
    .subscribe("daily.step.updates")
    .toFlowable()
    .filter(this::above10k)
    .distinct(KafkaConsumerRecord::key)
    .flatMapSingle(this::sendmail)
    .doOnError(err -> logger.error("Woops", err))
    .retryWhen(this::retryLater)
    .subscribe(mailResult -> logger.info("Congratulated {}", mailResult.getRecipients()));

  return Completable.complete();
}
 

@Override
public Completable rxStart() {
  Router router = Router.router(vertx);

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumerConfig("dashboard-webapp-throughput"))
    .subscribe("event-stats.throughput")
    .toFlowable()
    .subscribe(record -> forwardKafkaRecord(record, "client.updates.throughput"));

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumerConfig("dashboard-webapp-city-trend"))
    .subscribe("event-stats.city-trend.updates")
    .toFlowable()
    .subscribe(record -> forwardKafkaRecord(record, "client.updates.city-trend"));

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumerConfig("dashboard-webapp-ranking"))
    .subscribe("event-stats.user-activity.updates")
    .toFlowable()
    .filter(record -> record.value().getBoolean("makePublic"))
    .buffer(5, TimeUnit.SECONDS, RxHelper.scheduler(vertx))
    .subscribe(this::updatePublicRanking);

  hydrate();

  SockJSHandler sockJSHandler = SockJSHandler.create(vertx);
  SockJSBridgeOptions bridgeOptions = new SockJSBridgeOptions()
    .addInboundPermitted(new PermittedOptions().setAddressRegex("client.updates.*"))
    .addOutboundPermitted(new PermittedOptions().setAddressRegex("client.updates.*"));
  sockJSHandler.bridge(bridgeOptions);
  router.route("/eventbus/*").handler(sockJSHandler);

  router.get("/").handler(ctx -> ctx.reroute("/index.html"));
  router.route().handler(StaticHandler.create("webroot/assets"));

  return vertx.createHttpServer()
    .requestHandler(router)
    .rxListen(HTTP_PORT)
    .ignoreElement();
}
 

@BeforeEach
void setup(Vertx vertx, VertxTestContext testContext) {
  kafkaConsumer = KafkaConsumer.create(vertx, kafkaConfig());
  amqpClient = AmqpClient.create(vertx, amqClientOptions());
  KafkaAdminClient adminClient = KafkaAdminClient.create(vertx, kafkaConfig());
  vertx
    .rxDeployVerticle(new IngesterVerticle())
    .delay(500, TimeUnit.MILLISECONDS, RxHelper.scheduler(vertx))
    .flatMapCompletable(id -> adminClient.rxDeleteTopics(singletonList("incoming.steps")))
    .onErrorComplete()
    .subscribe(testContext::completeNow, testContext::failNow);
}
 

@Override
public Completable rxStart() {
  webClient = WebClient.create(vertx);
  producer = KafkaProducer.create(vertx, KafkaConfig.producer());

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumer("event-stats-throughput"))
    .subscribe("incoming.steps")
    .toFlowable()
    .buffer(5, TimeUnit.SECONDS, RxHelper.scheduler(vertx))
    .flatMapCompletable(this::publishThroughput)
    .doOnError(err -> logger.error("Woops", err))
    .retryWhen(this::retryLater)
    .subscribe();

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumer("event-stats-user-activity-updates"))
    .subscribe("daily.step.updates")
    .toFlowable()
    .flatMapSingle(this::addDeviceOwner)
    .flatMapSingle(this::addOwnerData)
    .flatMapCompletable(this::publishUserActivityUpdate)
    .doOnError(err -> logger.error("Woops", err))
    .retryWhen(this::retryLater)
    .subscribe();

  KafkaConsumer.<String, JsonObject>create(vertx, KafkaConfig.consumer("event-stats-city-trends"))
    .subscribe("event-stats.user-activity.updates")
    .toFlowable()
    .groupBy(this::city)
    .flatMap(groupedFlowable -> groupedFlowable.buffer(5, TimeUnit.SECONDS, RxHelper.scheduler(vertx)))
    .flatMapCompletable(this::publishCityTrendUpdate)
    .doOnError(err -> logger.error("Woops", err))
    .retryWhen(this::retryLater)
    .subscribe();

  return Completable.complete();
}
 

@BeforeEach
void prepare(Vertx vertx, VertxTestContext testContext) {
  producer = KafkaProducer.create(vertx, KafkaConfig.producer());
  consumer = KafkaConsumer.create(vertx, KafkaConfig.consumer(UUID.randomUUID().toString()));
  KafkaAdminClient adminClient = KafkaAdminClient.create(vertx, KafkaConfig.producer());
  adminClient
    .rxDeleteTopics(Arrays.asList("incoming.steps", "daily.step.updates"))
    .onErrorComplete()
    .andThen(vertx.rxDeployVerticle(new EventStatsVerticle()))
    .ignoreElement()
    .andThen(vertx.rxDeployVerticle(new FakeUserService()))
    .ignoreElement()
    .subscribe(testContext::completeNow, testContext::failNow);
}
 

private static void onStart() {
	System.err.println("Started");

	// Kafka setup for the example
    File dataDir = Testing.Files.createTestingDirectory("cluster");
    dataDir.deleteOnExit();
    KafkaCluster kafkaCluster;
	try {
		kafkaCluster = new KafkaCluster()
		  .usingDirectory(dataDir)
		  .withPorts(2181, 9092)
		  .addBrokers(1)
		  .deleteDataPriorToStartup(true)
		  .startup();
	} catch (IOException e) {
		throw new RuntimeException(e);
	}

    // Deploy the dashboard
    JsonObject consumerConfig = new JsonObject((Map) kafkaCluster.useTo()
      .getConsumerProperties("the_group", "the_client", OffsetResetStrategy.LATEST));

    AppGlobals globals = AppGlobals.get();
    
    // Create the consumer
	KafkaConsumer<String, JsonObject> consumer = KafkaConsumer.create(globals.getVertx(), (Map)consumerConfig.getMap(), 
    		String.class, JsonObject.class);
	
	BehaviorSubject<JsonObject> consumerReporter = BehaviorSubject.create();
	consumer.toObservable().subscribe(record -> consumerReporter.onNext(record.value()));
	
    // Subscribe to Kafka
    consumer.subscribe("the_topic");
    globals.setGlobal("consumer", consumerReporter);
    

    // Deploy the metrics collector : 3 times
    JsonObject producerConfig = new JsonObject((Map) kafkaCluster.useTo()
      .getProducerProperties("the_producer"));
    globals.getVertx().deployVerticle(
      MetricsVerticle.class.getName(),
      new DeploymentOptions().setConfig(producerConfig).setInstances(3)
    );
}