org.apache.flink.streaming.api.watermark.Watermark Java Examples

/**
 * Tests that for a multiple input valve, decreasing watermarks will yield no output.
 */
@Test
public void testMultipleInputDecreasingWatermarksYieldsNoOutput() {
	BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler();
	StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput);

	valve.inputWatermark(new Watermark(25), 0);
	valve.inputWatermark(new Watermark(10), 1);
	valve.inputWatermark(new Watermark(17), 2);
	assertEquals(new Watermark(10), valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(12), 0);
	valve.inputWatermark(new Watermark(8), 1);
	valve.inputWatermark(new Watermark(15), 2);
	assertEquals(null, valveOutput.popLastSeenOutput());
}
 

@SuppressWarnings("unchecked")
@Override
public void init() {
	// offer the queue for the tail
	BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
	LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);

	this.streamOutputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();

	// If timestamps are enabled we make sure to remove cyclic watermark dependencies
	if (isSerializingTimestamps()) {
		for (RecordWriterOutput<OUT> output : streamOutputs) {
			output.emitWatermark(new Watermark(Long.MAX_VALUE));
		}
	}
}
 

@Override
public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {

	ctx.emitWatermark(new Watermark(0));

	synchronized (ctx.getCheckpointLock()) {
		for (long i = 0; i < numElements; i++) {
			if (i % getRuntimeContext().getNumberOfParallelSubtasks() == getRuntimeContext().getIndexOfThisSubtask()) {
				ctx.collect(new Tuple2<>(i, i));
			}
		}
	}

	// don't emit a final watermark so that we don't trigger the registered event-time
	// timers
	while (isRunning) {
		Thread.sleep(20);
	}
}
 

@Test
public void testPop() {
	StreamElementQueue<Integer> queue = createStreamElementQueue(2);

	// add two elements to reach capacity
	putSuccessfully(queue, new Watermark(0L));
	ResultFuture<Integer> recordResult = putSuccessfully(queue, new StreamRecord<>(42, 1L));

	assertEquals(2, queue.size());

	// remove completed elements (watermarks are always completed)
	assertEquals(Arrays.asList(new Watermark(0L)), popCompleted(queue));
	assertEquals(1, queue.size());

	// now complete the stream record
	recordResult.complete(Collections.singleton(43));

	assertEquals(Arrays.asList(new StreamRecord<>(43, 1L)), popCompleted(queue));
	assertEquals(0, queue.size());
	assertTrue(queue.isEmpty());
}
 

@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark || o2 instanceof Watermark) {
		return 0;
	} else {
		StreamRecord<Tuple2<K, Integer>> sr0 = (StreamRecord<Tuple2<K, Integer>>) o1;
		StreamRecord<Tuple2<K, Integer>> sr1 = (StreamRecord<Tuple2<K, Integer>>) o2;
		if (sr0.getTimestamp() != sr1.getTimestamp()) {
			return (int) (sr0.getTimestamp() - sr1.getTimestamp());
		}
		int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
		if (comparison != 0) {
			return comparison;
		} else {
			return sr0.getValue().f1 - sr1.getValue().f1;
		}
	}
}
 

@Override
public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
	getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);

	// immediately trigger any set timers
	ctx.emitWatermark(new Watermark(1000));

	synchronized (ctx.getCheckpointLock()) {
		for (long i = 0; i < numElements; i++) {
			ctx.collect(new Tuple2<>(i, i));
		}
	}

	while (isRunning) {
		Thread.sleep(20);
	}
}
 

/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
private void emitRecordWithTimestampAndPunctuatedWatermark(
		T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
	@SuppressWarnings("unchecked")
	final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
			(KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;

	// only one thread ever works on accessing timestamps and watermarks
	// from the punctuated extractor
	final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
	final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);

	// emit the record with timestamp, using the usual checkpoint lock to guarantee
	// atomicity of record emission and offset state update
	synchronized (checkpointLock) {
		sourceContext.collectWithTimestamp(record, timestamp);
		partitionState.setOffset(offset);
	}

	// if we also have a new per-partition watermark, check if that is also a
	// new cross-partition watermark
	if (newWatermark != null) {
		updateMinPunctuatedWatermark(newWatermark);
	}
}
 

/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
private void emitRecordWithTimestampAndPunctuatedWatermark(
		T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
	@SuppressWarnings("unchecked")
	final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
			(KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;

	// only one thread ever works on accessing timestamps and watermarks
	// from the punctuated extractor
	final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
	final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);

	// emit the record with timestamp, using the usual checkpoint lock to guarantee
	// atomicity of record emission and offset state update
	synchronized (checkpointLock) {
		sourceContext.collectWithTimestamp(record, timestamp);
		partitionState.setOffset(offset);
	}

	// if we also have a new per-partition watermark, check if that is also a
	// new cross-partition watermark
	if (newWatermark != null) {
		updateMinPunctuatedWatermark(newWatermark);
	}
}
 

@Test
public void testPut() {
	StreamElementQueue<Integer> queue = createStreamElementQueue(2);

	Watermark watermark = new Watermark(0L);
	StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L);

	// add two elements to reach capacity
	assertTrue(queue.tryPut(watermark).isPresent());
	assertTrue(queue.tryPut(streamRecord).isPresent());

	assertEquals(2, queue.size());

	// queue full, cannot add new element
	assertFalse(queue.tryPut(new Watermark(2L)).isPresent());

	// check if expected values are returned (for checkpointing)
	assertEquals(Arrays.asList(watermark, streamRecord), queue.values());
}
 

/**
 * Feed a {@link Watermark} into the valve. If the input triggers the valve to output a new Watermark,
 * {@link ValveOutputHandler#handleWatermark(Watermark)} will be called to process the new Watermark.
 *
 * @param watermark the watermark to feed to the valve
 * @param channelIndex the index of the channel that the fed watermark belongs to (index starting from 0)
 */
public void inputWatermark(Watermark watermark, int channelIndex) {
	// ignore the input watermark if its input channel, or all input channels are idle (i.e. overall the valve is idle).
	if (lastOutputStreamStatus.isActive() && channelStatuses[channelIndex].streamStatus.isActive()) {
		long watermarkMillis = watermark.getTimestamp();

		// if the input watermark's value is less than the last received watermark for its input channel, ignore it also.
		if (watermarkMillis > channelStatuses[channelIndex].watermark) {
			channelStatuses[channelIndex].watermark = watermarkMillis;

			// previously unaligned input channels are now aligned if its watermark has caught up
			if (!channelStatuses[channelIndex].isWatermarkAligned && watermarkMillis >= lastOutputWatermark) {
				channelStatuses[channelIndex].isWatermarkAligned = true;
			}

			// now, attempt to find a new min watermark across all aligned channels
			findAndOutputNewMinWatermarkAcrossAlignedChannels();
		}
	}
}
 

@Override
public void emitWatermark(Watermark mark) {
	if (allowWatermark(mark)) {
		synchronized (checkpointLock) {
			streamStatusMaintainer.toggleStreamStatus(StreamStatus.ACTIVE);

			if (nextCheck != null) {
				this.failOnNextCheck = false;
			} else {
				scheduleNextIdleDetectionTask();
			}

			processAndEmitWatermark(mark);
		}
	}
}
 

@Test(timeout = 10_000)
public void testImpulseRestored() throws Exception {
  ImpulseSourceFunction source = new ImpulseSourceFunction(0);
  // Previous state available
  ListState<Object> mockListState = getMockListState(Collections.singletonList(true));
  source.initializeState(getInitializationContext(mockListState));

  // 1) Should finish
  source.run(sourceContext);
  // 2) Should keep checkpoint state
  verify(mockListState).get();
  verifyNoMoreInteractions(mockListState);
  // 3) Should always emit the final watermark
  verify(sourceContext).emitWatermark(Watermark.MAX_WATERMARK);
  // 4) Should _not_ emit impulse element
  verifyNoMoreInteractions(sourceContext);
}
 

@Override
 public void processWatermark(Watermark watermark) throws Exception {
   // we'll keep state forever in the operator

/*	StreamRecord<Tuple3<String, Long, Long>> result = new StreamRecord<>(null, -1);

	Iterator<Map.Entry<String, Map<Long, CountAndAccessTime>>> iterator = windows.entrySet().iterator();
	while (iterator.hasNext()) {
		Map.Entry<String, Map<Long, CountAndAccessTime>> campaignWindows = iterator.next();
		for(Map.Entry<Long, CountAndAccessTime> window: campaignWindows.getValue().entrySet()) {
			if(window.getKey() < watermark.getTimestamp() && window.getKey() >= lastWatermark) {
				// emit window
				Tuple3<String, Long, Long> resultTuple = Tuple3.of(campaignWindows.getKey(), window.getKey(), window.getValue().count);
				output.collect(result.replace(resultTuple));
			}
		}
	}
	lastWatermark = watermark.getTimestamp(); **/
 }
 

protected List<Watermark> extractWatermarks(Collection<Object> collection) {
	List<Watermark> watermarks = new ArrayList<>();
	for (Object obj : collection) {
		if (obj instanceof Watermark) {
			watermarks.add((Watermark) obj);
		}
	}
	return watermarks;
}
 

@Test
public void testEventTimeTimers() throws Exception {

	LegacyKeyedCoProcessOperator<String, Integer, String, String> operator =
			new LegacyKeyedCoProcessOperator<>(new EventTimeTriggeringProcessFunction());

	TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness =
			new KeyedTwoInputStreamOperatorTestHarness<>(
					operator,
					new IntToStringKeySelector<>(),
					new IdentityKeySelector<String>(),
					BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement1(new StreamRecord<>(17, 42L));
	testHarness.processElement2(new StreamRecord<>("18", 42L));

	testHarness.processWatermark1(new Watermark(5));
	testHarness.processWatermark2(new Watermark(5));

	testHarness.processWatermark1(new Watermark(6));
	testHarness.processWatermark2(new Watermark(6));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT1:17", 42L));
	expectedOutput.add(new StreamRecord<>("INPUT2:18", 42L));
	expectedOutput.add(new StreamRecord<>("1777", 5L));
	expectedOutput.add(new Watermark(5L));
	expectedOutput.add(new StreamRecord<>("1777", 6L));
	expectedOutput.add(new Watermark(6L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Override
public Watermark getCurrentWatermark() {
	// make sure timestamps are monotonously increasing, even when the system clock re-syncs
	final long now = Math.max(System.currentTimeMillis(), maxTimestamp);
	maxTimestamp = now;
	return new Watermark(now - 1);
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	super.processWatermark(mark);

	for (Watermark previousMark: watermarks) {
		assertTrue(previousMark.getTimestamp() < mark.getTimestamp());
	}
	watermarks.add(mark);
	latch.trigger();
	output.emitWatermark(mark);
}
 

@Override
public void processWatermark(Watermark mark) throws Exception {
	Optional<InternalTimeServiceManager<?>> timeServiceManager = getTimeServiceManager();
	if (timeServiceManager.isPresent()) {
		timeServiceManager.get().advanceWatermark(mark);
	}
	emitter.accept(mark);
}
 

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
	for (int i = 0; i < numWatermarks; i++) {
		ctx.collectWithTimestamp(i, initialTime + i);
		ctx.emitWatermark(new Watermark(initialTime + i));
	}
}
 

@Override
protected void partitionConsumerRecordsHandler(
		List<ConsumerRecord<byte[], byte[]>> partitionRecords,
		KafkaTopicPartitionState<T, TopicPartition> partition) throws Exception {

	for (ConsumerRecord<byte[], byte[]> record : partitionRecords) {
		final KafkaShuffleElement element = kafkaShuffleDeserializer.deserialize(record);

		// TODO: Do we need to check the end of stream if reaching the end watermark
		// TODO: Currently, if one of the partition sends an end-of-stream signal the fetcher stops running.
		// The current "ending of stream" logic in KafkaFetcher a bit strange: if any partition has a record
		// signaled as "END_OF_STREAM", the fetcher will stop running. Notice that the signal is coming from
		// the deserializer, which means from Kafka data itself. But it is possible that other topics
		// and partitions still have data to read. Finishing reading Partition0 can not guarantee that Partition1
		// also finishes.
		if (element.isRecord()) {
			// timestamp is inherent from upstream
			// If using ProcessTime, timestamp is going to be ignored (upstream does not include timestamp as well)
			// If using IngestionTime, timestamp is going to be overwritten
			// If using EventTime, timestamp is going to be used
			synchronized (checkpointLock) {
				KafkaShuffleRecord<T> elementAsRecord = element.asRecord();
				sourceContext.collectWithTimestamp(
					elementAsRecord.value,
					elementAsRecord.timestamp == null ? record.timestamp() : elementAsRecord.timestamp);
				partition.setOffset(record.offset());
			}
		} else if (element.isWatermark()) {
			final KafkaShuffleWatermark watermark = element.asWatermark();
			Optional<Watermark> newWatermark = watermarkHandler.checkAndGetNewWatermark(watermark);
			newWatermark.ifPresent(sourceContext::emitWatermark);
		}
	}
}
 

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}
 

@Test
public void testGroupedReduce() throws Exception {

	KeySelector<Integer, Integer> keySelector = new IntegerKeySelector();

	StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE);

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<>(1, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 3));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 4));
	testHarness.processElement(new StreamRecord<>(3, initialTime + 5));

	expectedOutput.add(new StreamRecord<>(1, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(3, initialTime + 5));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testBroadcastState() throws Exception {

	final Set<String> keysToRegister = new HashSet<>();
	keysToRegister.add("test1");
	keysToRegister.add("test2");
	keysToRegister.add("test3");

	try (
			TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(
					new TestFunction(keysToRegister), STATE_DESCRIPTOR)
	) {
		testHarness.processWatermark1(new Watermark(10L));
		testHarness.processWatermark2(new Watermark(10L));
		testHarness.processElement2(new StreamRecord<>(5, 12L));

		testHarness.processWatermark1(new Watermark(40L));
		testHarness.processWatermark2(new Watermark(40L));
		testHarness.processElement1(new StreamRecord<>("6", 13L));
		testHarness.processElement1(new StreamRecord<>("6", 15L));

		testHarness.processWatermark1(new Watermark(50L));
		testHarness.processWatermark2(new Watermark(50L));

		Queue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

		expectedOutput.add(new Watermark(10L));
		expectedOutput.add(new StreamRecord<>("5WM:10 TS:12", 12L));
		expectedOutput.add(new Watermark(40L));
		expectedOutput.add(new StreamRecord<>("6WM:40 TS:13", 13L));
		expectedOutput.add(new StreamRecord<>("6WM:40 TS:15", 15L));
		expectedOutput.add(new Watermark(50L));

		TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
	}
}
 

@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test(timeout = 10_000)
public void testKeepAlive() throws Exception {
  ImpulseSourceFunction source = new ImpulseSourceFunction(Long.MAX_VALUE);

  // No previous state available (=impulse should be emitted)
  ListState<Object> mockListState = getMockListState(Collections.emptyList());
  source.initializeState(getInitializationContext(mockListState));

  Thread sourceThread =
      new Thread(
          () -> {
            try {
              source.run(sourceContext);
              // should not finish
            } catch (Exception e) {
              LOG.error("Exception while executing ImpulseSourceFunction", e);
            }
          });
  try {
    sourceThread.start();
    source.cancel();
    // should finish
    sourceThread.join();
  } finally {
    sourceThread.interrupt();
    sourceThread.join();
  }
  verify(sourceContext).collect(argThat(elementMatcher));
  verify(sourceContext).emitWatermark(Watermark.MAX_WATERMARK);
  verify(mockListState).add(true);
  verify(mockListState).get();
  verifyNoMoreInteractions(mockListState);
}
 

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
	for (int i = 0; i < numWatermarks; i++) {
		ctx.collectWithTimestamp(i, initialTime + i);
		ctx.emitWatermark(new Watermark(initialTime + i));
	}
}
 

@Override
public void onProcessingTime(long timestamp) {
	final long currentTime = timeService.getCurrentProcessingTime();

	synchronized (lock) {
		// we should continue to automatically emit watermarks if we are active
		if (streamStatusMaintainer.getStreamStatus().isActive()) {
			if (idleTimeout != -1 && currentTime - lastRecordTime > idleTimeout) {
				// if we are configured to detect idleness, piggy-back the idle detection check on the
				// watermark interval, so that we may possibly discover idle sources faster before waiting
				// for the next idle check to fire
				markAsTemporarilyIdle();

				// no need to finish the next check, as we are now idle.
				cancelNextIdleDetectionTask();
			} else if (currentTime > nextWatermarkTime) {
				// align the watermarks across all machines. this will ensure that we
				// don't have watermarks that creep along at different intervals because
				// the machine clocks are out of sync
				final long watermarkTime = currentTime - (currentTime % watermarkInterval);

				output.emitWatermark(new Watermark(watermarkTime));
				nextWatermarkTime = watermarkTime + watermarkInterval;
			}
		}
	}

	long nextWatermark = currentTime + watermarkInterval;
	nextWatermarkTimer = this.timeService.registerTimer(
			nextWatermark, new WatermarkEmittingTask(this.timeService, lock, output));
}
 

@Override
public final Watermark getCurrentWatermark() {
	// this guarantees that the watermark never goes backwards.
	long potentialWM = currentMaxTimestamp - maxOutOfOrderness;
	if (potentialWM >= lastEmittedWatermark) {
		lastEmittedWatermark = potentialWM;
	}
	return new Watermark(lastEmittedWatermark);
}
 

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}
 

@Test
public void testGroupedReduce() throws Exception {

	KeySelector<Integer, Integer> keySelector = new IntegerKeySelector();

	StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE);

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<>(1, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 3));
	testHarness.processElement(new StreamRecord<>(2, initialTime + 4));
	testHarness.processElement(new StreamRecord<>(3, initialTime + 5));

	expectedOutput.add(new StreamRecord<>(1, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(2, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(3, initialTime + 5));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}