Java Code Examples for org.apache.flink.streaming.runtime.tasks.StreamTask#createRecordWriterDelegate()

public StreamSourceOperatorTestHarness(StreamSource<T, F> operator, int maxParallelism, int parallelism, int subtaskIndex) throws Exception {
    super(operator, maxParallelism, parallelism, subtaskIndex);
    this.sourceOperator = operator;
    this.triggeredCheckpoints = new ConcurrentLinkedQueue<>();
    this.operatorChain = new OperatorChain<>(this.mockTask, StreamTask.createRecordWriterDelegate(this.config, this.getEnvironment()));
}
 

private static OperatorChain<?, ?> createOperatorChain(AbstractStreamOperator<?> operator) {
  return new OperatorChain<>(
      operator.getContainingTask(),
      StreamTask.createRecordWriterDelegate(
          operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
}
 

private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	OperatorChain<?, ?> operatorChain = new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriterDelegate(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<>(output), operatorChain);
		operator.close();
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}
 

private <IN, OT extends StreamOperator<IN>> OperatorChain<IN, OT> createOperatorChain(
		StreamConfig streamConfig,
		Environment environment,
		StreamTask<IN, OT> task) {
	return new OperatorChain<>(task, StreamTask.createRecordWriterDelegate(streamConfig, environment));
}