org.apache.kafka.streams.processor.AbstractProcessor Java Examples

private static KafkaStreams createWikipediaStreamsInstance(String bootstrapServers) {
    final Serializer<JsonNode> jsonSerializer = new JsonSerializer();
    final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer();
    final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer);

    KStreamBuilder builder = new KStreamBuilder();
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "wikipedia-streams");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);


    KStream<JsonNode, JsonNode> wikipediaRaw = builder.stream(jsonSerde, jsonSerde, "wikipedia-raw");

    KStream<String, WikipediaMessage> wikipediaParsed =
            wikipediaRaw.map(WikipediaMessage::parceIRC)
                    .filter(WikipediaMessage::filterNonNull)
                    .through(Serdes.String(), new JsonPOJOSerde<>(WikipediaMessage.class), "wikipedia-parsed");

    KTable<String, Long> totalEditsByUser = wikipediaParsed
            .filter((key, value) -> value.type == WikipediaMessage.Type.EDIT)
            .countByKey(Serdes.String(), "wikipedia-edits-by-user");

    //some print
    totalEditsByUser.toStream().process(() -> new AbstractProcessor<String, Long>() {
        @Override
        public void process(String user, Long numEdits) {
            System.out.println("USER: " + user + " num.edits: " + numEdits);
        }
    });

    return new KafkaStreams(builder, props);

}
 

@Test
public void should_create_spans_from_stream_with_tracing_processor() {
  ProcessorSupplier<String, String> processorSupplier =
    kafkaStreamsTracing.processor(
      "forward-1", () ->
        new AbstractProcessor<String, String>() {
          @Override
          public void process(String key, String value) {
            try {
              Thread.sleep(100L);
            } catch (InterruptedException e) {
              e.printStackTrace();
            }
          }
        });

  String inputTopic = testName.getMethodName() + "-input";

  StreamsBuilder builder = new StreamsBuilder();
  builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String()))
    .process(processorSupplier);
  Topology topology = builder.build();

  KafkaStreams streams = buildKafkaStreams(topology);

  send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));

  waitForStreamToRun(streams);

  MutableSpan spanInput = testSpanHandler.takeRemoteSpan(CONSUMER);
  assertThat(spanInput.tags()).containsEntry("kafka.topic", inputTopic);

  MutableSpan spanProcessor = testSpanHandler.takeLocalSpan();
  assertChildOf(spanProcessor, spanInput);

  streams.close();
  streams.cleanUp();
}
 

@Test
public void should_create_spans_from_stream_without_tracing_and_tracing_processor() {
  ProcessorSupplier<String, String> processorSupplier =
    kafkaStreamsTracing.processor(
      "forward-1", () ->
        new AbstractProcessor<String, String>() {
          @Override
          public void process(String key, String value) {
            try {
              Thread.sleep(100L);
            } catch (InterruptedException e) {
              e.printStackTrace();
            }
          }
        });

  String inputTopic = testName.getMethodName() + "-input";

  StreamsBuilder builder = new StreamsBuilder();
  builder.stream(inputTopic, Consumed.with(Serdes.String(), Serdes.String()))
    .process(processorSupplier);
  Topology topology = builder.build();

  KafkaStreams streams = buildKafkaStreamsWithoutTracing(topology);

  send(new ProducerRecord<>(inputTopic, TEST_KEY, TEST_VALUE));

  waitForStreamToRun(streams);

  assertThat(testSpanHandler.takeLocalSpan().tags())
    .containsOnlyKeys("kafka.streams.application.id", "kafka.streams.task.id");

  streams.close();
  streams.cleanUp();
}
 

private void toEmail(KStream<Keys,MetricResult> result, ParameterOutput parameterOutput) {
    AbstractProcessor abstractProcessor = applicationContext.getBean(MetricsEmailProcessor.class, parameterOutput.getEmail(), parameterOutput.getTemplate());
    result.process(() -> abstractProcessor);
}
 

private void toSlack(KStream<Keys, MetricResult> result, ParameterOutput parameterOutput) {
    AbstractProcessor abstractProcessor = applicationContext.getBean(MetricsSlackProcessor.class, parameterOutput.getWebHookURL(), parameterOutput.getTemplate());
    result.process(() -> abstractProcessor);
}
 

protected void toElasticsearch(KStream<Keys, MetricResult> result, ParameterOutput parameterOutput) {
    AbstractProcessor abstractProcessor = applicationContext.getBean(MetricsElasticsearchProcessor.class, parameterOutput.getElasticsearchRetentionLevel(),parameterOutput.getIndexShape());
    result.process(() -> abstractProcessor);
}
 

protected void toSystemOut(KStream<Keys, MetricResult> result) {
    AbstractProcessor abstractProcessor = applicationContext.getBean(LoggingProcessor.class);
    result.process(() -> abstractProcessor);
}
 

private void toSnmp(KStream<Keys, MetricResult> result, ParameterOutput parameterOutput) {
    AbstractProcessor abstractProcessor = applicationContext.getBean(MetricsSnmpProcessor.class);
    result.process(() -> abstractProcessor);
}
 

/**
 * Create a foreach processor, similar to {@link KStream#foreach(ForeachAction)}, where its action
 * will be recorded in a new span with the indicated name.
 *
 * <p>Simple example using Kafka Streams DSL:
 * <pre>{@code
 * StreamsBuilder builder = new StreamsBuilder();
 * builder.stream(inputTopic)
 *        .process(kafkaStreamsTracing.foreach("myForeach", (k, v) -> ...);
 * }</pre>
 */
public <K, V> ProcessorSupplier<K, V> foreach(String spanName, ForeachAction<K, V> action) {
  return new TracingProcessorSupplier<>(this, spanName, () ->
    new AbstractProcessor<K, V>() {
      @Override public void process(K key, V value) {
        action.apply(key, value);
      }
    });
}