org.apache.kafka.streams.kstream.Windowed Java Examples

public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.inputTopic);

    final KTable<Windowed<Integer>, Long> counts = clickEvents
            .groupByKey()
            .windowedBy(TimeWindows.of(Duration.ofMinutes(1)))
            .count();

    counts.toStream()
            .map((key, value) -> KeyValue.pair(
                    key.key(),
                    new ClickOutput(key.key(), value, key.window().start())))
            .to(this.outputTopic, Produced.with(Serdes.Integer(), new JsonSerde<>(ClickOutput.class)));

    return builder.build();
}
 

private static String printWindowedKey(Config config, ConsumerRecord<Windowed<String>, Long> windowedKeyValue) {

        return String.format("Count = %s for Key = %s, at window [%s-%s] %s (%s)",
                windowedKeyValue.value().toString(),
                windowedKeyValue.key().key(),
                DateTimeFormatter
                        .ofPattern("HH:mm:ss")
                        .withLocale(Locale.getDefault())
                        .withZone(ZoneId.systemDefault())
                        .format(windowedKeyValue.key().window().startTime()),
                DateTimeFormatter
                        .ofPattern("HH:mm:ss")
                        .withLocale(Locale.getDefault())
                        .withZone(ZoneId.systemDefault())
                        .format(windowedKeyValue.key().window().endTime()),
                DateTimeFormatter
                        .ofPattern(config.getString("local.date.pattern"))
                        .withLocale(Locale.forLanguageTag(config.getString("local.date.lang")))
                        .withZone(ZoneId.systemDefault())
                        .format(windowedKeyValue.key().window().startTime()),
                ZoneId.systemDefault().getId()
        );
    }
 

@Override
public void start(Future<Void> startFuture) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "kiqr");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.LongSerde.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0");

    KStreamBuilder builder = new KStreamBuilder();
    KTable<String, Long> table = builder.table(Serdes.String(), Serdes.Long(), "visits", "visitStore");
    KTable<Windowed<String>, Long> windowedCount = table.toStream().groupByKey().count(TimeWindows.of(60), "visitCount");





    vertx.deployVerticle(RestKiqrServerVerticle.Builder.serverBuilder(builder, props).withPort(2901).build(), res -> {
        if (res.succeeded()) {
            startFuture.complete();
        } else {
            startFuture.fail(res.cause());
        }
    });
}
 

@Override
public void start() throws Exception {

    execute(Config.SESSION_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> {

        KeyBasedQuery query = (KeyBasedQuery) abstractQuery;
        ReadOnlySessionStore<Object, Object> store = streams.store(query.getStoreName(), QueryableStoreTypes.sessionStore());
        try (KeyValueIterator<Windowed<Object>, Object> result = store.fetch(deserializeObject(keySerde, query.getKey()))) {

            if (result.hasNext()) {
                List<Window> results = new ArrayList<>();
                while (result.hasNext()) {


                    KeyValue<Windowed<Object>, Object> windowedEntry = result.next();
                    results.add(new Window(windowedEntry.key.window().start(), windowedEntry.key.window().end(), base64Encode(valueSerde, windowedEntry.value)));
                }
                return new SessionQueryResponse(results);
            } else {
                return new SessionQueryResponse(Collections.emptyList());
            }
        }
    });


}
 

private Predicate getWindowedKeyPredicate() {
  final ExpressionMetadata expressionEvaluator = createExpressionMetadata();
  return (Predicate<Windowed<String>, GenericRow>) (key, row) -> {
    try {
      Kudf[] kudfs = expressionEvaluator.getUdfs();
      Object[] values = new Object[columnIndexes.length];
      for (int i = 0; i < values.length; i++) {
        if (columnIndexes[i] < 0) {
          values[i] = kudfs[i];
        } else {
          values[i] = genericRowValueTypeEnforcer
              .enforceFieldType(
                  columnIndexes[i],
                  row.getColumns().get(columnIndexes[i]
                  )
              );
        }
      }
      return (Boolean) ee.evaluate(values);
    } catch (Exception e) {
      log.error(e.getMessage(), e);
    }
    log.error("Invalid format: " + key + " : " + row);
    return false;
  };
}
 

@Override
public void apply(K key, GenericRow row) {
  try {
    if (row == null) {
      return;
    }
    if (limit.isPresent()) {
      counter++;
      if (counter > limit.get()) {
        throw new KsqlException("LIMIT reached for the partition.");
      }
    }
    String keyString;
    if (key instanceof Windowed) {
      Windowed windowedKey = (Windowed) key;
      keyString = String.format("%s : %s", windowedKey.key(), windowedKey.window());
    } else {
      keyString = Objects.toString(key);
    }
    queue.put(new KeyValue<>(keyString, row));
  } catch (InterruptedException exception) {
    throw new KsqlException("InterruptedException while enqueueing:" + key);
  }
}
 

@Test
public void shouldComputeMedian() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 2}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 3}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 4}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 5}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 6}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 100}")
    );


    String destTopic = "median-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("median", destTopic), "median-src", functionRegistry, udafRegistry) {

        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("median");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {
            return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    KafkaUnit.Message<Keys, MetricResult> result1 = resultInDestTopic.get(0);
    assertThat(result1.getKey().getRuleName()).isEqualTo("median");
    assertThat(result1.getKey().getRuleDSL()).isNotBlank();
    assertThat(result1.getKey().getProject()).isEqualTo("myproject");
    assertThat(result1.getValue().getResult()).isEqualTo(4.0029296875);
}
 

@SuppressWarnings("unchecked")
@Override
public KTable<Windowed<Short>, T> run(final KStream<Edge<K>, EV> edgeStream) {

    //For parallel window support we key the edge stream by partition and apply a parallel fold per partition.
    //Finally, we merge all locally combined results into our final graph aggregation property.
    KTable<Windowed<Short>, S> partialAgg = edgeStream
        .groupByKey(Grouped.with(new KryoSerde<>(), new KryoSerde<>()))
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .aggregate(this::initialValue, new PartialAgg<>(updateFun()))
        .toStream()
        .groupBy((k, v) -> GLOBAL_KEY)
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .reduce(combineFun())
        .mapValues(aggregator(edgeStream), Materialized.<Windowed<Short>, S, KeyValueStore<Bytes, byte[]>>
            as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>()));

    if (transform() != null) {
        return partialAgg.mapValues(
            transform(),
            Materialized.<Windowed<Short>, T, KeyValueStore<Bytes, byte[]>>
                as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>())
        );
    }

    return (KTable<Windowed<Short>, T>) partialAgg;
}
 

@Test
public void test() throws Exception {

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
        LongSerializer.class, new Properties()
    );
    StreamsBuilder builder = new StreamsBuilder();

    KStream<Edge<Long>, Void> edges = StreamUtils.streamFromCollection(builder, producerConfig, new KryoSerde<>(),
        new KryoSerde<>(), getEdges()
    );
    KGraphStream<Long, Void, Void> graph =
        new EdgeStream<>(edges, GraphSerialized.with(new KryoSerde<>(), new KryoSerde<>(), new KryoSerde<>()));

    KTable<Windowed<Short>, DisjointSet<Long>> sets = graph.aggregate(new ConnectedComponents<>(500L));

    startStreams(builder, new KryoSerde<>(), new KryoSerde<>());

    Thread.sleep(10000);

    List<String> values = StreamUtils.listFromTable(streams, sets).stream()
        .map(kv -> kv.value.toString())
        .collect(Collectors.toList());

    // verify the results
    String expectedResultStr = "1, 2, 3, 5\n" + "6, 7\n" + "8, 9\n";
    String[] result = parser(values);
    String[] expected = expectedResultStr.split("\n");

    assertEquals("Different number of lines in expected and obtained result.", expected.length, result.length);
    Assert.assertArrayEquals("Different connected components.", expected, result);

    streams.close();
}
 

@Test
public void testBipartite() throws Exception {

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
        LongSerializer.class, new Properties()
    );
    StreamsBuilder builder = new StreamsBuilder();

    KStream<Edge<Long>, Void> edges = StreamUtils.streamFromCollection(builder, producerConfig, new KryoSerde<>(),
        new KryoSerde<>(), getBipartiteEdges()
    );
    KGraphStream<Long, Void, Void> graph =
        new EdgeStream<>(edges, GraphSerialized.with(new KryoSerde<>(), new KryoSerde<>(), new KryoSerde<>()));

    KTable<Windowed<Short>, Candidates> candidates = graph.aggregate(new BipartitenessCheck<>(500L));

    startStreams(builder, new KryoSerde<>(), new KryoSerde<>());

    Thread.sleep(10000);

    List<String> result = StreamUtils.listFromTable(streams, candidates).stream()
        .map(kv -> kv.value.toString())
        .collect(Collectors.toList());

    // verify the results
    assertEquals(
        Lists.newArrayList(
            "(true,{1={1=(1,true), 2=(2,false), 3=(3,false), 4=(4,false), 5=(5,true), 7=(7,true), 9=(9,true)}})"),
        result
    );

    streams.close();
}
 

@Test
public void testNonBipartite() throws Exception {

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
        LongSerializer.class, new Properties()
    );
    StreamsBuilder builder = new StreamsBuilder();

    KStream<Edge<Long>, Void> edges = StreamUtils.streamFromCollection(builder, producerConfig, new KryoSerde<>(),
        new KryoSerde<>(), getNonBipartiteEdges()
    );
    KGraphStream<Long, Void, Void> graph =
        new EdgeStream<>(edges, GraphSerialized.with(new KryoSerde<>(), new KryoSerde<>(), new KryoSerde<>()));

    KTable<Windowed<Short>, Candidates> candidates = graph.aggregate(new BipartitenessCheck<>(500L));

    startStreams(builder, new KryoSerde<>(), new KryoSerde<>());

    Thread.sleep(10000);

    List<String> result = StreamUtils.listFromTable(streams, candidates).stream()
        .map(kv -> kv.value.toString())
        .collect(Collectors.toList());

    // verify the results
    assertEquals(
        Lists.newArrayList(
            "(false,{})"),
        result
    );

    streams.close();
}
 

private String fetchFromSessionStore(Map<String, String> params) {
    String store = params.get(STORE_PARAM);
    String key = params.get(KEY_PARAM);

    HostInfo storeHostInfo = getHostInfo(store, key);

    if (storeHostInfo.host().equals("unknown")) {
        return STORES_NOT_ACCESSIBLE;
    }

    if (dataNotLocal(storeHostInfo)) {
        LOG.info("{} located in state store on another instance !!!!", key);
        return fetchRemote(storeHostInfo, "session", params);
    }

    ReadOnlySessionStore<String, CustomerTransactions> readOnlySessionStore = kafkaStreams.store(store, QueryableStoreTypes.sessionStore());
    List<String> results = new ArrayList<>();
    List<KeyValue<String, List<String>>> sessionResults = new ArrayList<>();
    try (KeyValueIterator<Windowed<String>, CustomerTransactions> iterator = readOnlySessionStore.fetch(key)) {
        while (iterator.hasNext()) {
            KeyValue<Windowed<String>, CustomerTransactions> windowed = iterator.next();
            CustomerTransactions transactions = windowed.value;
            LocalDateTime startSession = getLocalDateTime(windowed.key.window().start());
            LocalDateTime endSession = getLocalDateTime(windowed.key.window().end());
            transactions.setSessionInfo(String.format("Session Window{start=%s, end=%s}", startSession.toLocalTime().toString(), endSession.toLocalTime().toString()));
            results.add(transactions.toString());
        }
        sessionResults.add(new KeyValue<>(key, results));
    }

    return gson.toJson(sessionResults);
}
 

@Test
public void test() throws Exception {

    Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
        LongSerializer.class, new Properties()
    );
    StreamsBuilder builder = new StreamsBuilder();

    // Use 1 partition for deterministic ordering
    int numPartitions = 1;
    KStream<Edge<Long>, Void> edges = StreamUtils.streamFromCollection(builder, producerConfig,
        "temp-" + UUID.randomUUID(), numPartitions, (short) 1, new KryoSerde<>(), new KryoSerde<>(), getEdges()
    );
    KGraphStream<Long, Void, Void> graph =
        new EdgeStream<>(edges, GraphSerialized.with(new KryoSerde<>(), new KryoSerde<>(), new KryoSerde<>()));

    KTable<Windowed<Short>, AdjacencyListGraph<Long>> sets = graph.aggregate(new Spanner<>(mergeWindowTime, k));

    startStreams(builder, new KryoSerde<>(), new KryoSerde<>());

    Thread.sleep(10000);

    List<String> values = StreamUtils.listFromTable(streams, sets).stream()
        .map(kv -> kv.value.toString())
        .collect(Collectors.toList());

    // This result will vary depending on the number of partitions
    assertEquals(
        "[{1=[4], 2=[3], 3=[2, 4], 4=[1, 3, 5, 7], 5=[4, 6], 6=[5, 8], 7=[4, 8], 8=[6, 7, 9], 9=[8]}]",
        values.toString()
    );

    streams.close();
}
 

@SuppressWarnings("unchecked")
public <T> T key() {
  if (window == null) {
    return (T) key;
  }
  return (T) new Windowed<>(key, new TimeWindow(window.start, window.end));
}
 

public MetricResult(Windowed<Keys> keysWindowed, Double result) {
    this.ruleName = keysWindowed.key().getRuleName();
    this.ruleDSL = keysWindowed.key().getRuleDSL();
    this.project = keysWindowed.key().getProject();
    this.keys = Collections.unmodifiableMap(keysWindowed.key().getKeys());
    this.startDate = new Date(keysWindowed.window().start());
    this.endDate = new Date(keysWindowed.window().end());
    //for Elasticsearch
    this.timestamp = endDate;
    this.result = result;
    this.element = null;

}
 

public static void main(String[] args) {

        StreamsConfig streamingConfig = new StreamsConfig(getProperties());

        JsonSerializer<StockTransactionCollector> stockTransactionsSerializer = new JsonSerializer<>();
        JsonDeserializer<StockTransactionCollector> stockTransactionsDeserializer = new JsonDeserializer<>(StockTransactionCollector.class);
        JsonDeserializer<StockTransaction> stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class);
        JsonSerializer<StockTransaction> stockTxnJsonSerializer = new JsonSerializer<>();
        Serde<StockTransaction> transactionSerde = Serdes.serdeFrom(stockTxnJsonSerializer,stockTxnDeserializer);
        StringSerializer stringSerializer = new StringSerializer();
        StringDeserializer stringDeserializer = new StringDeserializer();
        Serde<String> stringSerde = Serdes.serdeFrom(stringSerializer,stringDeserializer);
        Serde<StockTransactionCollector> collectorSerde = Serdes.serdeFrom(stockTransactionsSerializer,stockTransactionsDeserializer);
        WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerializer);
        WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringDeserializer);
        Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer,windowedDeserializer);

        KStreamBuilder kStreamBuilder = new KStreamBuilder();


        KStream<String,StockTransaction> transactionKStream =  kStreamBuilder.stream(stringSerde,transactionSerde,"stocks");

        transactionKStream.map((k,v)-> new KeyValue<>(v.getSymbol(),v))
                          .through(stringSerde, transactionSerde,"stocks-out")
                          .groupBy((k,v) -> k, stringSerde, transactionSerde)
                          .aggregate(StockTransactionCollector::new,
                               (k, v, stockTransactionCollector) -> stockTransactionCollector.add(v),
                               TimeWindows.of(10000),
                               collectorSerde, "stock-summaries")
                .to(windowedSerde,collectorSerde,"transaction-summary");


        System.out.println("Starting StockStreams Example");
        KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamingConfig);
        kafkaStreams.start();
        System.out.println("Now started StockStreams Example");

    }
 

@Test
public void shouldComputeSum() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 9}")
    );


    String destTopic = "sum-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("sum", destTopic), "sum-src", functionRegistry, udafRegistry) {

        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("sum");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {
            return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    KafkaUnit.Message<Keys, MetricResult> result1 = resultInDestTopic.get(0);
    assertThat(result1.getKey().getRuleName()).isEqualTo("sum");
    assertThat(result1.getKey().getRuleDSL()).isNotBlank();
    assertThat(result1.getKey().getProject()).isEqualTo("myproject");
    assertThat(result1.getValue().getResult()).isEqualTo(10);
}
 

@Test
public void shouldComputeAvg() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 9}")
    );


    String destTopic = "avg-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("avg", destTopic), "avg-src", functionRegistry, udafRegistry) {

        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("avg");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {
            return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    KafkaUnit.Message<Keys, MetricResult> result1 = resultInDestTopic.get(0);
    assertThat(result1.getKey().getRuleName()).isEqualTo("avg");
    assertThat(result1.getKey().getRuleDSL()).isNotBlank();
    assertThat(result1.getKey().getProject()).isEqualTo("myproject");
    assertThat(result1.getValue().getResult()).isEqualTo(5);
}
 

@Test
public void shouldComputeMax() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 10}")
    );
    String destTopic = "max-dest";

    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("max", destTopic), "max-src", functionRegistry, udafRegistry) {

        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("max");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {
            return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    KafkaUnit.Message<Keys, MetricResult> result1 = resultInDestTopic.get(0);
    assertThat(result1.getKey().getRuleName()).isEqualTo("max");
    assertThat(result1.getKey().getRuleDSL()).isNotBlank();
    assertThat(result1.getKey().getProject()).isEqualTo("myproject");
    assertThat(result1.getValue().getResult()).isEqualTo(10);
}
 

@Test
public void shouldComputeMin() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 10}")
    );
    String destTopic = "min-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("min", destTopic), "min-src", functionRegistry, udafRegistry) {
        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("min");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {
            return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    assertThat(resultInDestTopic.get(0).getKey().getRuleName()).isEqualTo("min");
    assertThat(resultInDestTopic.get(0).getKey().getRuleDSL()).isNotBlank();
    assertThat(resultInDestTopic.get(0).getKey().getProject()).isEqualTo("myproject");
    assertThat(resultInDestTopic.get(0).getValue().getResult()).isEqualTo(1);
}
 

@Test
public void shouldComputeCountDistinct() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 10}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 11}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 12}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 13}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"anotherone\",\"duration\": 13}")

    );
    String destTopic = "count-distinct-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("count-distinct", destTopic), "count-distinct-src", functionRegistry, udafRegistry) {
        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("count-distinct");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {                return value.path("type");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    assertThat(resultInDestTopic.get(0).getKey().getRuleName()).isEqualTo("count-distinct");
    assertThat(resultInDestTopic.get(0).getKey().getRuleDSL()).isNotBlank();
    assertThat(resultInDestTopic.get(0).getKey().getProject()).isEqualTo("myproject");
    assertThat(resultInDestTopic.get(0).getValue().getResult()).isEqualTo(3);
}
 

@Test
public void shouldComputeCount() {

    List<JsonNode> input = Arrays.asList(
            toJsonNode("{\"project\":\"myproject\",\"type\":\"something\",\"duration\": 1}"),
            toJsonNode("{\"project\":\"myproject\",\"type\":\"somethingelse\",\"duration\": 10}")
    );
    String destTopic = "count-dest";
    GenericMetricProcessor minDuration = new GenericMetricProcessor(buildProcessMetric("count", destTopic), "count-src", functionRegistry, udafRegistry) {
        @Override
        protected AggregateFunction aggInitializer() {
            return aggFunction("count");
        }

        @Override
        protected KTable<Windowed<Keys>, Double> aggregate(KGroupedStream<Keys, JsonNode> kGroupedStream) {
            return aggregateTumblingWindow(kGroupedStream, 1, TimeUnit.SECONDS);
        }

        @Override
        protected JsonNode mapValues(JsonNode value) {                return value.path("duration");
        }
    };
    List<KafkaUnit.Message<Keys, MetricResult>> resultInDestTopic = executeMetricStream(input, minDuration, destTopic);
    assertThat(resultInDestTopic).hasSize(1);
    assertThat(resultInDestTopic.get(0).getKey().getRuleName()).isEqualTo("count");
    assertThat(resultInDestTopic.get(0).getKey().getRuleDSL()).isNotBlank();
    assertThat(resultInDestTopic.get(0).getKey().getProject()).isEqualTo("myproject");
    assertThat(resultInDestTopic.get(0).getValue().getResult()).isEqualTo(2);
}
 

private List<ProducerRecord<Windowed<String>, Long>> readAtLeastNOutputs(int size) {
    List<ProducerRecord<Windowed<String>, Long>> result = new ArrayList<>();

    Assert.assertThat(() -> {
        ProducerRecord<Windowed<String>, Long> record = readNext();
        if(null != record) result.add(record);
        return result.size();
    }, eventuallyEval(is(size), this.testGracePeriodSize));

    return result;
}
 

private ProducerRecord<Windowed<String>, Long> readNext() {
    return topologyTestDriver.readOutput(
            this.outputTopic,
            this.keyResultSerde.deserializer(),
            Serdes.Long().deserializer()
    );
}
 

public static void main(String[] args) {

        final Config config = ConfigFactory.load();
        final String outputTopic = config.getString("output.topic.name");

        final ActorSystem system = ActorSystem.create();
        final Materializer materializer = ActorMaterializer.create(system);

        final ConsumerSettings<Windowed<String>, Long> consumerSettings =
                ConsumerSettings
                        .create(
                                system,
                                timeWindowedSerdeFrom(
                                        String.class,
                                        config.getDuration("window.size").toMillis()
                                ).deserializer(),
                                Serdes.Long().deserializer()
                        )
                        .withGroupId(UUID.randomUUID().toString())
                        .withBootstrapServers(config.getString("bootstrap.servers"))
                        .withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

        Consumer.plainSource(
                consumerSettings,
                Subscriptions.topics(outputTopic))
                .to(Sink.foreach((record) -> {
                            logger.info(printWindowedKey(config, record));
                            return BoxedUnit.UNIT;
                        })
                ).run(materializer);

    }
 

public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();

    // Click Events
    final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.clickInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)));

    final KTable<Windowed<Integer>, Long> counts = clickEvents
            .selectKey(((key, value) -> value.getStatus()))
            .filter(((key, value) -> key >= 400))
            .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)))
            .windowedBy(TimeWindows.of(Duration.ofMinutes(1)))  // 1 Minute in ms
            .count();

    // Status codes
    final KTable<Integer, StatusCode> statusCodes = builder.table(this.statusInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(StatusCode.class)));

    // Join
    final KStream<Integer, ErrorOutput> errors = counts.toStream()
            .map((key, value) -> KeyValue.pair(
                    key.key(),
                    new ErrorOutput(key.key(), value, key.window().start(), null /*empty definition*/)))
            .join(statusCodes,
                    (countRecord, code) -> new ErrorOutput(
                            countRecord.getStatusCode(), countRecord.getCount(), countRecord.getTime(), code.getDefinition()),
                    Joined.valueSerde(new JsonSerde<>(ErrorOutput.class)));
    errors.to(this.errorOutputTopic);

    // Send alert if more than 5x a certain error code per minute
    errors.filter((key, errorOutput) -> errorOutput.getCount() > 5L).to(this.alertTopic);

    return builder.build();
}
 

@Get("/dependencies")
public AggregatedHttpResponse getDependencies(
    @Param("endTs") long endTs,
    @Param("lookback") long lookback
) {
  try {
    if (!storage.dependencyQueryEnabled) return AggregatedHttpResponse.of(HttpStatus.NOT_FOUND);
    ReadOnlyWindowStore<Long, DependencyLink> store =
        storage.getDependencyStorageStream()
            .store(DEPENDENCIES_STORE_NAME, QueryableStoreTypes.windowStore());
    List<DependencyLink> links = new ArrayList<>();
    Instant from = Instant.ofEpochMilli(endTs - lookback);
    Instant to = Instant.ofEpochMilli(endTs);
    try (KeyValueIterator<Windowed<Long>, DependencyLink> iterator = store.fetchAll(from, to)) {
      iterator.forEachRemaining(keyValue -> links.add(keyValue.value));
    }
    List<DependencyLink> mergedLinks = DependencyLinker.merge(links);
    LOG.debug("Dependencies found from={}-to={}: {}", from, to, mergedLinks.size());
    return AggregatedHttpResponse.of(
        HttpStatus.OK,
        MediaType.JSON,
        DependencyLinkBytesEncoder.JSON_V1.encodeList(mergedLinks));
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    return AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE);
  }
}
 

/** Runs the streams program, writing to the "long-counts-all" topic.
 *
 * @param args Not used.
 */
public static void main(String[] args) throws Exception {
    
    Properties config = new Properties();

    config.put(StreamsConfig.APPLICATION_ID_CONFIG,
        "tumbling-window-kafka-streams");
    config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
        "localhost:9092");
    config.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG,
        "localhost:2181");
    config.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG,
        Serdes.ByteArray().getClass().getName());
    config.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG,
        Serdes.Long().getClass().getName());

    KStreamBuilder builder = new KStreamBuilder();

    KStream<byte[], Long> longs = builder.stream(
        Serdes.ByteArray(), Serdes.Long(), "longs");

    // The tumbling windows will clear every ten seconds.
    KTable<Windowed<byte[]>, Long> longCounts =
        longs.groupByKey()
             .count(TimeWindows.of(10000L)
                               .until(10000L),
                    "long-counts");

    // Write to topics.
    longCounts.toStream((k,v) -> k.key())
              .to(Serdes.ByteArray(),
                  Serdes.Long(),
                  "long-counts-all");

    KafkaStreams streams = new KafkaStreams(builder, config);
    streams.start();

    // Now generate the data and write to the topic.
    Properties producerConfig = new Properties();
    producerConfig.put("bootstrap.servers", "localhost:9092");
    producerConfig.put("key.serializer",
                       "org.apache.kafka.common" +
                       ".serialization.ByteArraySerializer");
    producerConfig.put("value.serializer",
                       "org.apache.kafka.common" +
                       ".serialization.LongSerializer");

    KafkaProducer producer = 
        new KafkaProducer<byte[], Long>(producerConfig);

    Random rng = new Random(12345L);

    while(true) { 
        producer.send(new ProducerRecord<byte[], Long>(
            "longs", "A".getBytes(), rng.nextLong()%10));
        Thread.sleep(500L);
    } // Close infinite data generating loop.
}
 

@Override
public KeyValue<Windowed<Object>, Object> next() {
    return iterator.next();
}
 

@Override
public Windowed<Object> peekNextKey() {
    throw new UnsupportedOperationException();
}