Java Code Examples for org.apache.kafka.streams.Topology#describe()

@Test void should_doNothing_whenAllDisabled() {
  // Given: configs
  Duration traceTtl = Duration.ofMillis(5);
  Duration traceTtlCheckInterval = Duration.ofMinutes(1);
  List<String> autocompleteKeys = Collections.singletonList("environment");
  // When: topology provided
  Topology topology = new TraceStorageTopology(
      spansTopic,
      autocompleteKeys,
      traceTtl,
      traceTtlCheckInterval,
      0,
      false,
      false).get();
  TopologyDescription description = topology.describe();
  // Then:
  assertThat(description.subtopologies()).hasSize(0);
  // Given: streams config
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  testDriver.close();
}
 

@Test void should_doNothing_whenDisabled() {
  // Given: configs
  Duration dependenciesRetentionPeriod = Duration.ofMinutes(1);
  Duration dependenciesWindowSize = Duration.ofMillis(100);
  // When: topology created
  Topology topology = new DependencyStorageTopology(
      dependencyTopic,
      dependenciesRetentionPeriod,
      dependenciesWindowSize,
      false).get();
  TopologyDescription description = topology.describe();
  // Then: topology with 1 thread
  assertThat(description.subtopologies()).hasSize(0);
  // Given: streams configuration
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  testDriver.close();
}
 

@Test void should_doNothing_whenAggregationDisabled() {
  Duration traceTimeout = Duration.ofSeconds(1);
  Topology topology = new SpanAggregationTopology(
      spansTopic,
      traceTopic,
      dependencyTopic,
      traceTimeout,
      false).get();
  TopologyDescription description = topology.describe();
  // Then: single threaded topology
  assertThat(description.subtopologies()).hasSize(0);
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  testDriver.close();
}
 

static TopologyDescription.Node getNodeByName(final Topology topology, final String nodeName) {
  final TopologyDescription description = topology.describe();
  final Set<TopologyDescription.Subtopology> subtopologies = description.subtopologies();
  List<TopologyDescription.Node> nodes = subtopologies.stream().flatMap(subtopology -> subtopology.nodes().stream()).collect(Collectors.toList());
  final Map<String, List<TopologyDescription.Node>> nodesByName = nodes.stream().collect(Collectors.groupingBy(TopologyDescription.Node::name));
  return nodesByName.get(nodeName).get(0);
}
 

@Test void should_persistSpans_and_onlyQueryTraces_whenEnabled() {
  // Given: configs
  Duration traceTtl = Duration.ofMillis(5);
  Duration traceTtlCheckInterval = Duration.ofMinutes(1);
  List<String> autocompleteKeys = Collections.singletonList("environment");
  SpansSerde spansSerde = new SpansSerde();
  // When: topology provided
  Topology topology = new TraceStorageTopology(
      spansTopic,
      autocompleteKeys,
      traceTtl,
      traceTtlCheckInterval,
      0,
      true,
      false).get();
  TopologyDescription description = topology.describe();
  // Then: 1 thread prepared
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams config
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .timestamp(10000L).duration(10L)
      .build();
  Span c = Span.newBuilder().traceId("c").id("c").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  List<Span> spans = Arrays.asList(a, b, c);
  testDriver.pipeInput(factory.create(spansTopic, a.traceId(), spans, 10L));
  // Then: trace stores are filled
  KeyValueStore<String, List<Span>> traces = testDriver.getKeyValueStore(TRACES_STORE_NAME);
  assertThat(traces.get(a.traceId())).containsExactlyElementsOf(spans);
  KeyValueStore<Long, Set<String>> spanIdsByTs =
      testDriver.getKeyValueStore(SPAN_IDS_BY_TS_STORE_NAME);
  KeyValueIterator<Long, Set<String>> ids = spanIdsByTs.all();
  assertThat(ids).hasNext();
  assertThat(ids.next().value).containsExactly(a.traceId());
  // Then: service name stores are filled
  KeyValueStore<String, String> serviceNames =
      testDriver.getKeyValueStore(SERVICE_NAMES_STORE_NAME);
  assertThat(serviceNames).isNull();
  KeyValueStore<String, Set<String>> spanNames =
      testDriver.getKeyValueStore(SPAN_NAMES_STORE_NAME);
  assertThat(spanNames).isNull();
  KeyValueStore<String, Set<String>> autocompleteTags =
      testDriver.getKeyValueStore(AUTOCOMPLETE_TAGS_STORE_NAME);
  assertThat(autocompleteTags).isNull();
  // Finally close resources
  testDriver.close();
  spansSerde.close();
}
 

@Test void should_persistSpans_and_searchQueryTraces_whenAllEnabled() {
  // Given: configs
  Duration traceTtl = Duration.ofMillis(5);
  Duration traceTtlCheckInterval = Duration.ofMinutes(1);
  List<String> autocompleteKeys = Collections.singletonList("environment");
  SpansSerde spansSerde = new SpansSerde();
  // When: topology provided
  Topology topology = new TraceStorageTopology(
      spansTopic,
      autocompleteKeys,
      traceTtl,
      traceTtlCheckInterval,
      0,
      true,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: 1 thread prepared
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams config
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .timestamp(10000L).duration(10L)
      .build();
  Span c = Span.newBuilder().traceId("c").id("c").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .timestamp(10000L).duration(11L)
      .putTag("environment", "dev")
      .build();
  List<Span> spans = Arrays.asList(a, b, c);
  testDriver.pipeInput(factory.create(spansTopic, a.traceId(), spans, 10L));
  // Then: trace stores are filled
  KeyValueStore<String, List<Span>> traces = testDriver.getKeyValueStore(TRACES_STORE_NAME);
  assertThat(traces.get(a.traceId())).containsExactlyElementsOf(spans);
  KeyValueStore<Long, Set<String>> spanIdsByTs =
      testDriver.getKeyValueStore(SPAN_IDS_BY_TS_STORE_NAME);
  KeyValueIterator<Long, Set<String>> ids = spanIdsByTs.all();
  assertThat(ids).hasNext();
  assertThat(ids.next().value).containsExactly(a.traceId());
  // Then: service name stores are filled
  KeyValueStore<String, String> serviceNames =
      testDriver.getKeyValueStore(SERVICE_NAMES_STORE_NAME);
  List<String> serviceNameList = new ArrayList<>();
  serviceNames.all().forEachRemaining(serviceName -> serviceNameList.add(serviceName.value));
  assertThat(serviceNameList).hasSize(2);
  assertThat(serviceNames.get("svc_a")).isEqualTo("svc_a");
  assertThat(serviceNames.get("svc_b")).isEqualTo("svc_b");
  KeyValueStore<String, Set<String>> spanNames =
      testDriver.getKeyValueStore(SPAN_NAMES_STORE_NAME);
  assertThat(spanNames.get("svc_a")).containsExactly("op_a");
  assertThat(spanNames.get("svc_b")).containsExactly("op_b");
  KeyValueStore<String, Set<String>> autocompleteTags =
      testDriver.getKeyValueStore(AUTOCOMPLETE_TAGS_STORE_NAME);
  assertThat(autocompleteTags.get("environment")).containsExactly("dev");
  // When: clock moves forward
  Span d = Span.newBuilder()
      .traceId("d")
      .id("d")
      .timestamp(
          MILLISECONDS.toMicros(traceTtlCheckInterval.toMillis()) + MILLISECONDS.toMicros(20))
      .build();
  testDriver.pipeInput(
      factory.create(spansTopic, d.traceId(), Collections.singletonList(d),
          traceTtlCheckInterval.plusMillis(1).toMillis()));
  // Then: Traces store is empty
  assertThat(traces.get(a.traceId())).isNull();
  // Finally close resources
  testDriver.close();
  spansSerde.close();
}
 

@Test void should_aggregateSpans_and_mapDependencies() {
  // Given: configuration
  Duration traceTimeout = Duration.ofSeconds(1);
  SpansSerde spansSerde = new SpansSerde();
  DependencyLinkSerde dependencyLinkSerde = new DependencyLinkSerde();
  // When: topology built
  Topology topology = new SpanAggregationTopology(
      spansTopic,
      traceTopic,
      dependencyTopic,
      traceTimeout,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: single threaded topology
  assertThat(description.subtopologies()).hasSize(1);
  // Given: test driver
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: two related spans coming on the same Session window
  ConsumerRecordFactory<String, List<Span>> factory =
      new ConsumerRecordFactory<>(spansTopic, new StringSerializer(), spansSerde.serializer());
  Span a = Span.newBuilder().traceId("a").id("a").name("op_a").kind(Span.Kind.CLIENT)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_a").build())
      .build();
  Span b = Span.newBuilder().traceId("a").id("b").name("op_b").kind(Span.Kind.SERVER)
      .localEndpoint(Endpoint.newBuilder().serviceName("svc_b").build())
      .build();
  testDriver.pipeInput(
      factory.create(spansTopic, a.traceId(), Collections.singletonList(a), 0L));
  testDriver.pipeInput(
      factory.create(spansTopic, b.traceId(), Collections.singletonList(b), 0L));
  // When: and new record arrive, moving the event clock further than inactivity gap
  Span c = Span.newBuilder().traceId("c").id("c").build();
  testDriver.pipeInput(factory.create(spansTopic, c.traceId(), Collections.singletonList(c),
      traceTimeout.toMillis() + 1));
  // Then: a trace is aggregated.1
  ProducerRecord<String, List<Span>> trace =
      testDriver.readOutput(traceTopic, new StringDeserializer(), spansSerde.deserializer());
  assertThat(trace).isNotNull();
  OutputVerifier.compareKeyValue(trace, a.traceId(), Arrays.asList(a, b));
  // Then: a dependency link is created
  ProducerRecord<String, DependencyLink> linkRecord =
      testDriver.readOutput(dependencyTopic, new StringDeserializer(),
          dependencyLinkSerde.deserializer());
  assertThat(linkRecord).isNotNull();
  DependencyLink link = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  OutputVerifier.compareKeyValue(linkRecord, "svc_a:svc_b", link);
  //Finally close resources
  testDriver.close();
  spansSerde.close();
  dependencyLinkSerde.close();
}
 

@Test void should_storeDependencies() {
  // Given: configs
  Duration dependenciesRetentionPeriod = Duration.ofMinutes(1);
  Duration dependenciesWindowSize = Duration.ofMillis(100);
  // When: topology created
  Topology topology = new DependencyStorageTopology(
      dependencyTopic,
      dependenciesRetentionPeriod,
      dependenciesWindowSize,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: topology with 1 thread
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams configuration
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, DependencyLink> factory =
      new ConsumerRecordFactory<>(dependencyTopic, new StringSerializer(),
          dependencyLinkSerde.serializer());
  DependencyLink dependencyLink = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  String dependencyLinkId = "svc_a:svc_b";
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 10L));
  WindowStore<String, DependencyLink> links = testDriver.getWindowStore(DEPENDENCIES_STORE_NAME);
  // Then: dependency link created
  WindowStoreIterator<DependencyLink> firstLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(firstLink).hasNext();
  assertThat(firstLink.next().value).isEqualTo(dependencyLink);
  // When: new links appear
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 90L));
  // Then: dependency link increases
  WindowStoreIterator<DependencyLink> secondLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(secondLink).hasNext();
  assertThat(secondLink.next().value.callCount()).isEqualTo(2);
  // When: time moves forward
  testDriver.advanceWallClockTime(dependenciesRetentionPeriod.toMillis() + 91L);
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink));
  // Then: dependency link is removed and restarted
  KeyValueIterator<Windowed<String>, DependencyLink> thirdLink = links.all();
  assertThat(thirdLink).hasNext();
  assertThat(thirdLink.next().value.callCount()).isEqualTo(1);
  // Close resources
  testDriver.close();
  dependencyLinkSerde.close();
}