org.apache.samza.task.MessageCollector Java Examples

@Test
@SuppressWarnings("unchecked")
public void testAsyncFlatMapOperator() {
  AsyncFlatMapOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockOp = mock(AsyncFlatMapOperatorSpec.class);
  AsyncFlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> txfmFn = mock(AsyncFlatMapFunction.class);
  when(mockOp.getTransformFn()).thenReturn(txfmFn);
  AsyncFlatmapOperatorImpl<TestMessageEnvelope, TestOutputMessageEnvelope> opImpl =
      new AsyncFlatmapOperatorImpl<>(mockOp);
  TestMessageEnvelope inMsg = mock(TestMessageEnvelope.class);
  Collection<TestOutputMessageEnvelope> mockOutputs = mock(Collection.class);
  when(txfmFn.apply(inMsg)).thenReturn(CompletableFuture.supplyAsync(() -> mockOutputs));
  MessageCollector mockCollector = mock(MessageCollector.class);
  TaskCoordinator mockCoordinator = mock(TaskCoordinator.class);
  Collection<TestOutputMessageEnvelope> results = opImpl
      .handleMessage(inMsg, mockCollector, mockCoordinator);
  verify(txfmFn, times(1)).apply(inMsg);
  assertEquals(results, mockOutputs);
}
 

@Override
protected CompletionStage<Collection<JM>> handleMessageAsync(M message, MessageCollector collector,
    TaskCoordinator coordinator) {
  Collection<JM> output = Collections.emptyList();

  try {
    KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState();
    KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState();

    K key = thisPartialJoinFn.getKey(message);
    thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis()));
    TimestampedValue<OM> otherMessage = otherState.get(key);

    long now = clock.currentTimeMillis();
    if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) {
      JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue());
      output = Collections.singletonList(joinResult);
    }
  } catch (Exception e) {
    throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e);
  }

  return CompletableFuture.completedFuture(output);
}
 

@Override
public void window(MessageCollector collector,
  TaskCoordinator coordinator) {

  KeyValueIterator<String, String> entries = store.all();
  while (entries.hasNext()) {                                        // c
    Entry<String, String> entry = entries.next();
    String key = entry.getKey();
    String value = entry.getValue();
    if (isTimestampKey(key) && Cart.isAbandoned(value)) {            // d
      String shopper = extractShopper(key);
      String cart = store.get(asCartKey(shopper));
      
      AbandonedCartEvent event =
        new AbandonedCartEvent(shopper, cart);
      collector.send(new OutgoingMessageEnvelope(
        new SystemStream("kafka", "derived-events-ch04"), event));    // e
      
      resetShopper(shopper);
    }
  }
}
 

private void processWatermark(long timeStamp, MessageCollector collector) {
    if (timeStamp > lastWatermark + watermarkEvictionPeriod) {
        for (SlicingWindowOperator<Value> slicingWindowOperator : this.slicingWindowOperatorMap.values()) {
            List<AggregateWindow> aggregates = slicingWindowOperator.processWatermark(timeStamp);
            for (AggregateWindow<Value> aggregateWindow : aggregates) {
                if (aggregateWindow.hasValue()) {
                    System.out.println(aggregateWindow);
                    for (Value aggValue : aggregateWindow.getAggValues()) {
                        collector.send(new OutgoingMessageEnvelope(outputStream, aggValue));
                    }

                }
            }
        }
        lastWatermark = timeStamp;
    }
}
 

public Optional<WindowPane<K, Object>> onMessage(TriggerKey<K> triggerKey, M message,
    MessageCollector collector, TaskCoordinator coordinator) {
  if (!isCancelled) {
    LOG.trace("Forwarding callbacks for {}", message);
    impl.onMessage(message, triggerScheduler);

    if (impl.shouldFire()) {
      // repeating trigger can trigger multiple times, So, clear the state to allow future triggerings.
      if (impl instanceof RepeatingTriggerImpl) {
        ((RepeatingTriggerImpl<M, K>) impl).clear();
      }
      return onTriggerFired(triggerKey, collector, coordinator);
    }
  }
  return Optional.empty();
}
 

@Override
protected CompletionStage<Collection<Object>> handleMessageAsync(IncomingMessageEnvelope message,
    MessageCollector collector, TaskCoordinator coordinator) {
  Object result;
  InputTransformer transformer = inputOpSpec.getTransformer();
  if (transformer != null) {
    result = transformer.apply(message);
  } else {
    result = this.inputOpSpec.isKeyed() ? KV.of(message.getKey(), message.getMessage()) : message.getMessage();
  }

  Collection<Object> output = Optional.ofNullable(result)
      .map(Collections::singletonList)
      .orElse(Collections.emptyList());

  return CompletableFuture.completedFuture(output);
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  if (envelope.getSystemStreamPartition().getStream().equals("epoch")) {
    int newEpoch = Integer.parseInt((String) envelope.getMessage());
    logger.info("New epoch in message - " + newEpoch);

    Integer epoch = getInt(EPOCH);
    if (epoch == null || newEpoch == epoch)
      return;
    if (newEpoch < epoch)
      throw new IllegalArgumentException("Got new epoch " + newEpoch + " which is less than current epoch " + epoch);

    // it's a new era, reset current epoch and count
    logger.info("Epoch: " + newEpoch);
    this.state.put(EPOCH, Integer.toString(newEpoch));
    this.state.put(COUNT, "0");
    coordinator.commit(RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

private void fanOut(String sender, Map<String, Object> message, MessageCollector collector) {
  // Colon is used as separator, and semicolon is lexicographically after colon
  KeyValueIterator<String, String> followers = socialGraph.range(sender + ":", sender + ";");

  try {
    while (followers.hasNext()) {
      String[] follow = followers.next().getKey().split(":");
      if (!follow[0].equals(sender)) {
        throw new IllegalStateException("Social graph db prefix doesn't match: " + sender + " != " + follow[0]);
      }
      message.put("recipient", follow[1]);
      collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.DELIVERIES_STREAM, follow[1], null, message));
    }
  } finally {
    followers.close();
  }
}
 

@Test
public void joinRetainsLatestMessageForKey() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to first stream
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to first stream again with same keys but different values
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, 2 * n), messageCollector, taskCoordinator, taskCallback));
  // push messages to second stream with same key
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  int outputSum = output.stream().reduce(0, (s, m) -> s + m);
  assertEquals(165, outputSum); // should use latest messages in the first stream
}
 

@Test
public void joinRetainsLatestMessageForKeyReverse() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to second stream
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to second stream again with same keys but different values
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator, taskCallback));
  // push messages to first stream with same key
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  int outputSum = output.stream().reduce(0, (s, m) -> s + m);
  assertEquals(165, outputSum); // should use latest messages in the second stream
}
 

@Test
public void joinRetainsMatchedMessagesReverse() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to first stream
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to second stream with same key
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  int outputSum = output.stream().reduce(0, (s, m) -> s + m);
  assertEquals(110, outputSum);

  output.clear();

  // push messages to second stream with same keys once again.
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  int newOutputSum = output.stream().reduce(0, (s, m) -> s + m);
  assertEquals(110, newOutputSum); // should produce the same output as before
}
 

@Test
public void joinRemovesExpiredMessages() throws Exception {
  TestClock testClock = new TestClock();
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(testClock, streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to first stream
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1))); // 1 minute after ttl
  sot.window(messageCollector, taskCoordinator); // should expire first stream messages

  // push messages to second stream with same key
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  assertTrue(output.isEmpty());
}
 

@Test
public void joinRemovesExpiredMessagesReverse() throws Exception {
  TestClock testClock = new TestClock();
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(testClock, streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to second stream
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1))); // 1 minute after ttl
  sot.window(messageCollector, taskCoordinator); // should expire second stream messages

  // push messages to first stream with same key
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  assertTrue(output.isEmpty());
}
 

@Test
public void testTumblingAggregatingWindowsDiscardingMode() throws Exception {
  when(this.context.getTaskContext().getStore("jobName-jobId-window-w1"))
      .thenReturn(new TestInMemoryStore<>(new TimeSeriesKeySerde(new IntegerSerde()), new IntegerSerde()));

  OperatorSpecGraph sgb = this.getAggregateTumblingWindowStreamGraph(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph();
  List<WindowPane<Integer, Integer>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
  task.init(this.context);
  MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Integer>) envelope.getMessage());
  integers.forEach(n -> task.processAsync(new IntegerEnvelope(n), messageCollector, taskCoordinator, taskCallback));
  testClock.advanceTime(Duration.ofSeconds(1));

  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 5);
  Assert.assertEquals(windowPanes.get(0).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(1).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(2).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(3).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(4).getMessage(), new Integer(1));
}
 

@Test
public void testEndOfStreamFlushesWithNoTriggerFirings() throws Exception {

  OperatorSpecGraph sgb =
      this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph();
  TestClock testClock = new TestClock();
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
  StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
  task.init(this.context);

  MessageCollector messageCollector =
    envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());

  task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
  task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
  task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);
  task.processAsync(new IntegerEnvelope(1), messageCollector, taskCoordinator, taskCallback);

  final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(
      new SystemStreamPartition("kafka", "integers", new Partition(0)));
  task.processAsync(endOfStream, messageCollector, taskCoordinator, taskCallback);
  Assert.assertEquals(windowPanes.size(), 1);
  Assert.assertEquals(windowPanes.get(0).getMessage().size(), 4);
  verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
  verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}
 

@Test
public void testOnMessageUpdatesMetrics() {
  ReadableMetricsRegistry mockMetricsRegistry = mock(ReadableMetricsRegistry.class);
  when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(mockMetricsRegistry);
  Counter mockCounter = mock(Counter.class);
  Timer mockTimer = mock(Timer.class);
  when(mockMetricsRegistry.newCounter(anyString(), anyString())).thenReturn(mockCounter);
  when(mockMetricsRegistry.newTimer(anyString(), anyString())).thenReturn(mockTimer);

  Object mockTestOpImplOutput = mock(Object.class);
  OperatorImpl<Object, Object> opImpl = new TestOpImpl(mockTestOpImplOutput);
  opImpl.init(this.internalTaskContext);

  // send a message to this operator
  MessageCollector mockCollector = mock(MessageCollector.class);
  TaskCoordinator mockCoordinator = mock(TaskCoordinator.class);
  opImpl.onMessage(mock(Object.class), mockCollector, mockCoordinator);

  // verify that it updates message count and timer metrics
  verify(mockCounter, times(1)).inc();
  verify(mockTimer, times(1)).update(anyLong());
}
 

@Test
public void testWithUnkeyedInput() {
  InputOperatorImpl inputOperator =
      new InputOperatorImpl(new InputOperatorSpec("stream-id", null, null, null, false, "input-op-id"));

  IncomingMessageEnvelope ime =
      new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg");

  Collection<Object> results =
      inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class));

  Object result = results.iterator().next();
  assertEquals("msg", result);
}
 

@Test
public void testTumblingWindowsDiscardingMode() throws Exception {
  OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph();
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
  task.init(this.context);
  MessageCollector messageCollector =
    envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  integers.forEach(n -> task.processAsync(new IntegerEnvelope(n), messageCollector, taskCoordinator, taskCallback));
  testClock.advanceTime(Duration.ofSeconds(1));

  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 5);
  Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(4).getKey().getKey(), new Integer(3));
  Assert.assertEquals((windowPanes.get(4).getMessage()).size(), 1);
}
 

@Override
public void apply(M message, MessageCollector messageCollector, TaskCoordinator taskCoordinator) {
  actual.add(message);

  if (actual.size() >= expected.size()) {
    timerTask.cancel();
    check();
  }
}
 

@Test
public void testBroadcast() {
  SystemStreamMetadata metadata = mock(SystemStreamMetadata.class);
  Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>();
  partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata);
  StreamMetadataCache metadataCache = mock(StreamMetadataCache.class);
  when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata);

  SystemStream systemStream = new SystemStream("test-system", "test-stream");
  Set<Integer> partitions = new HashSet<>();
  MessageCollector collector = mock(MessageCollector.class);
  doAnswer(invocation -> {
    OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0];
    partitions.add((Integer) envelope.getPartitionKey());
    assertEquals(envelope.getSystemStream(), systemStream);
    return null;
  }).when(collector).send(any());

  ControlMessageSender sender = new ControlMessageSender(metadataCache);
  WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0");
  SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0));
  sender.broadcastToOtherPartitions(watermark, ssp, collector);
  assertEquals(partitions.size(), 3);
}
 

private void processMessageEvent(Map<String, Object> message, MessageCollector collector) {
  if (!message.get("event").equals("postMessage")) {
    throw new IllegalStateException("Unexpected event type on messages stream: " + message.get("event"));
  }
  String sender = (String) message.get("sender");
  String time = (String) message.get("time");

  userTimeline.put(sender + ":" + time + ":" + numMessages, message);
  numMessages++;
  fanOut(sender, message, collector);
}
 

@Test
public void testSend() {
  SystemStreamMetadata metadata = mock(SystemStreamMetadata.class);
  Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>();
  partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class));
  when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata);
  StreamMetadataCache metadataCache = mock(StreamMetadataCache.class);
  when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata);

  SystemStream systemStream = new SystemStream("test-system", "test-stream");
  Set<Integer> partitions = new HashSet<>();
  MessageCollector collector = mock(MessageCollector.class);
  doAnswer(invocation -> {
    OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0];
    partitions.add((Integer) envelope.getPartitionKey());
    assertEquals(envelope.getSystemStream(), systemStream);
    return null;
  }).when(collector).send(any());

  ControlMessageSender sender = new ControlMessageSender(metadataCache);
  WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0");
  sender.send(watermark, systemStream, collector);
  assertEquals(partitions.size(), 1);
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  String key = (String) envelope.getKey();
  String value = (String) envelope.getMessage();
  String[] pieces = value.split("-");
  int epoch = Integer.parseInt(pieces[0]);

  int partition = Integer.parseInt(pieces[1].split(" ")[1]);
  Partitions partitions = loadPartitions(epoch, key);
  logger.info("Joiner got epoch = " + epoch + ", partition = " + partition + ", parts = " + partitions);
  if (partitions.epoch < epoch) {
    // we are in a new era
    if (partitions.partitions.size() != expected)
      throw new IllegalArgumentException("Should have " + expected + " partitions when new epoch starts.");
    logger.info("Reseting epoch to " + epoch);
    this.store.delete(key);
    partitions.epoch = epoch;
    partitions.partitions.clear();
    partitions.partitions.add(partition);
  } else if (partitions.epoch > epoch) {
    logger.info("Ignoring message for epoch " + epoch);
  } else {
    partitions.partitions.add(partition);
    if (partitions.partitions.size() == expected) {
      logger.info("Completed: " + key + " -> " + Integer.toString(epoch));
      collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "completed-keys"), key, Integer.toString(epoch)));
    }
  }
  this.store.put(key, partitions.toString());
  logger.info("Join store in Task " + this.taskName + " " + key + " -> " + partitions.toString());
}
 

@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  int epoch = Integer.parseInt((String) envelope.getMessage());
  if (epoch > currentEpoch) {
    logger.info("Epoch changed to " + epoch + " from " + currentEpoch);
    this.currentEpoch = epoch;
    this.lastEpochChange = System.currentTimeMillis();
    this.inError = false;
  }
}
 

@Override
public void processAsync(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator,
    final TaskCallback callback) {
  // Mimic a random callback delay ans send message
  RestCall call = new RestCall(envelope, collector, callback);
  call.start();
}
 

@Test
public void joinNoMatchReverse() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to second stream
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to first stream with different keys
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n + 100, n), messageCollector, taskCoordinator, taskCallback));

  assertTrue(output.isEmpty());
}
 

@Test
public void joinNoMatch() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to first stream
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to second stream with different keys
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n + 100, n), messageCollector, taskCoordinator, taskCallback));

  assertTrue(output.isEmpty());
}
 

@Test
public void joinReverse() throws Exception {
  StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction());
  StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc);
  List<Integer> output = new ArrayList<>();
  MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage());

  // push messages to second stream
  numbers.forEach(n -> sot.processAsync(new SecondStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));
  // push messages to first stream with same keys
  numbers.forEach(n -> sot.processAsync(new FirstStreamIME(n, n), messageCollector, taskCoordinator, taskCallback));

  int outputSum = output.stream().reduce(0, (s, m) -> s + m);
  assertEquals(110, outputSum);
}
 

/**
 * Wraps {@code storeToWrap} into a {@link LoggedStore} if {@code changelogSSP} is defined.
 * Otherwise, returns the original {@code storeToWrap}.
 */
private static KeyValueStore<byte[], byte[]> buildMaybeLoggedStore(SystemStreamPartition changelogSSP,
    String storeName,
    MetricsRegistry registry,
    StoreProperties.StorePropertiesBuilder storePropertiesBuilder,
    KeyValueStore<byte[], byte[]> storeToWrap,
    MessageCollector changelogCollector) {
  if (changelogSSP == null) {
    return storeToWrap;
  } else {
    LoggedStoreMetrics loggedStoreMetrics = new LoggedStoreMetrics(storeName, registry);
    storePropertiesBuilder.setLoggedStore(true);
    return new LoggedStore<>(storeToWrap, changelogSSP, changelogCollector, loggedStoreMetrics);
  }
}
 

@Test
public void testTumblingWindowsAccumulatingMode() throws Exception {
  OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.ACCUMULATING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph();
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, testClock);
  task.init(this.context);

  MessageCollector messageCollector =
    envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  integers.forEach(n -> task.processAsync(new IntegerEnvelope(n), messageCollector, taskCoordinator, taskCallback));
  testClock.advanceTime(Duration.ofSeconds(1));
  task.window(messageCollector, taskCoordinator);

  Assert.assertEquals(windowPanes.size(), 7);
  Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4);

  Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4);
}