org.apache.flink.streaming.runtime.io.RecordWriterOutput Java Examples

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@SuppressWarnings("unchecked")
@Override
public void init() {
	// offer the queue for the tail
	BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
	LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);

	this.streamOutputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();

	// If timestamps are enabled we make sure to remove cyclic watermark dependencies
	if (isSerializingTimestamps()) {
		for (RecordWriterOutput<OUT> output : streamOutputs) {
			output.emitWatermark(new Watermark(Long.MAX_VALUE));
		}
	}
}
 

@VisibleForTesting
OperatorChain(
		List<StreamOperatorWrapper<?, ?>> allOperatorWrappers,
		RecordWriterOutput<?>[] streamOutputs,
		WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainEntryPoint,
		StreamOperatorWrapper<OUT, OP> headOperatorWrapper) {

	this.streamOutputs = checkNotNull(streamOutputs);
	this.chainEntryPoint = checkNotNull(chainEntryPoint);
	this.operatorEventDispatcher = null;

	checkState(allOperatorWrappers != null && allOperatorWrappers.size() > 0);
	this.headOperatorWrapper = checkNotNull(headOperatorWrapper);
	this.tailOperatorWrapper = allOperatorWrappers.get(0);
	this.numOperators = allOperatorWrappers.size();

	linkOperatorWrappers(allOperatorWrappers);
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@SuppressWarnings("unchecked")
@Override
public void init() {
	// offer the queue for the tail
	BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
	LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);

	this.streamOutputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();

	// If timestamps are enabled we make sure to remove cyclic watermark dependencies
	if (isSerializingTimestamps()) {
		synchronized (getCheckpointLock()) {
			for (RecordWriterOutput<OUT> output : streamOutputs) {
				output.emitWatermark(new Watermark(Long.MAX_VALUE));
			}
		}
	}
}
 

@Override
protected void performDefaultAction(DefaultActionContext context) throws Exception {
	StreamRecord<OUT> nextRecord = shouldWait ?
		dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS) :
		dataChannel.take();

	if (nextRecord != null) {
		synchronized (getCheckpointLock()) {
			for (RecordWriterOutput<OUT> output : streamOutputs) {
				output.collect(nextRecord);
			}
		}
	} else {
		context.allActionsCompleted();
	}
}
 

@Override
public void toggleStreamStatus(StreamStatus status) {
	if (!status.equals(this.streamStatus)) {
		this.streamStatus = status;

		// try and forward the stream status change to all outgoing connections
		for (RecordWriterOutput<?> streamOutput : streamOutputs) {
			streamOutput.emitStreamStatus(status);
		}
	}
}
 

private <IN, OUT> WatermarkGaugeExposingOutput<StreamRecord<IN>> createChainedOperator(
		StreamTask<?, ?> containingTask,
		StreamConfig operatorConfig,
		Map<Integer, StreamConfig> chainedConfigs,
		ClassLoader userCodeClassloader,
		Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
		List<StreamOperator<?>> allOperators,
		OutputTag<IN> outputTag) {
	// create the output that the operator writes to first. this may recursively create more operators
	WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainedOperatorOutput = createOutputCollector(
		containingTask,
		operatorConfig,
		chainedConfigs,
		userCodeClassloader,
		streamOutputs,
		allOperators);

	// now create the operator and give it the output collector to write its output to
	StreamOperatorFactory<OUT> chainedOperatorFactory = operatorConfig.getStreamOperatorFactory(userCodeClassloader);
	OneInputStreamOperator<IN, OUT> chainedOperator = chainedOperatorFactory.createStreamOperator(
			containingTask, operatorConfig, chainedOperatorOutput);

	allOperators.add(chainedOperator);

	WatermarkGaugeExposingOutput<StreamRecord<IN>> currentOperatorOutput;
	if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
		currentOperatorOutput = new ChainingOutput<>(chainedOperator, this, outputTag);
	}
	else {
		TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader);
		currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
	}

	// wrap watermark gauges since registered metrics must be unique
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue);
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);

	return currentOperatorOutput;
}
 

@Override
protected void processInput(MailboxDefaultAction.Controller controller) throws Exception {
	StreamRecord<OUT> nextRecord = shouldWait ?
		dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS) :
		dataChannel.take();

	if (nextRecord != null) {
		for (RecordWriterOutput<OUT> output : streamOutputs) {
			output.collect(nextRecord);
		}
	} else {
		controller.allActionsCompleted();
	}
}
 

@Override
public void toggleStreamStatus(StreamStatus status) {
	if (!status.equals(this.streamStatus)) {
		this.streamStatus = status;

		// try and forward the stream status change to all outgoing connections
		for (RecordWriterOutput<?> streamOutput : streamOutputs) {
			streamOutput.emitStreamStatus(status);
		}
	}
}
 

@VisibleForTesting
OperatorChain(
		StreamOperator<?>[] allOperators,
		RecordWriterOutput<?>[] streamOutputs,
		WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainEntryPoint,
		OP headOperator) {

	this.allOperators = checkNotNull(allOperators);
	this.streamOutputs = checkNotNull(streamOutputs);
	this.chainEntryPoint = checkNotNull(chainEntryPoint);
	this.headOperator = checkNotNull(headOperator);
}
 

@SafeVarargs
private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(
		OneInputStreamOperator<T, T>... operators) {

	checkNotNull(operators);
	checkArgument(operators.length > 0);

	try (MockEnvironment env = MockEnvironment.builder().build()) {

	final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env);

		final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class);
		final StreamConfig cfg = new StreamConfig(new Configuration());

		final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length];

		// initial output goes to nowhere
		@SuppressWarnings({"unchecked", "rawtypes"})
		WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(
				new Output[0], statusProvider);

		// build the reverse operators array
		for (int i = 0; i < ops.length; i++) {
			OneInputStreamOperator<T, T> op = operators[ops.length - i - 1];
			op.setup(containingTask, cfg, lastWriter);
			lastWriter = new ChainingOutput<>(op, statusProvider, null);
			ops[i] = op;
		}

		@SuppressWarnings("unchecked")
		final OP head = (OP) operators[0];

		return new OperatorChain<>(
				ops,
				new RecordWriterOutput<?>[0],
				lastWriter,
				head);
	}
}
 

private <IN, OUT> WatermarkGaugeExposingOutput<StreamRecord<IN>> createChainedOperator(
		StreamTask<?, ?> containingTask,
		StreamConfig operatorConfig,
		Map<Integer, StreamConfig> chainedConfigs,
		ClassLoader userCodeClassloader,
		Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
		List<StreamOperator<?>> allOperators,
		OutputTag<IN> outputTag) {
	// create the output that the operator writes to first. this may recursively create more operators
	WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainedOperatorOutput = createOutputCollector(
		containingTask,
		operatorConfig,
		chainedConfigs,
		userCodeClassloader,
		streamOutputs,
		allOperators);

	// now create the operator and give it the output collector to write its output to
	OneInputStreamOperator<IN, OUT> chainedOperator = operatorConfig.getStreamOperator(userCodeClassloader);

	chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput);

	allOperators.add(chainedOperator);

	WatermarkGaugeExposingOutput<StreamRecord<IN>> currentOperatorOutput;
	if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
		currentOperatorOutput = new ChainingOutput<>(chainedOperator, this, outputTag);
	}
	else {
		TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader);
		currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
	}

	// wrap watermark gauges since registered metrics must be unique
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue);
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);

	return currentOperatorOutput;
}
 

@Override
public void toggleStreamStatus(StreamStatus status) {
	if (!status.equals(this.streamStatus)) {
		this.streamStatus = status;

		// try and forward the stream status change to all outgoing connections
		for (RecordWriterOutput<?> streamOutput : streamOutputs) {
			streamOutput.emitStreamStatus(status);
		}
	}
}
 

@VisibleForTesting
OperatorChain(
		StreamOperator<?>[] allOperators,
		RecordWriterOutput<?>[] streamOutputs,
		WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainEntryPoint,
		OP headOperator) {

	this.allOperators = checkNotNull(allOperators);
	this.streamOutputs = checkNotNull(streamOutputs);
	this.chainEntryPoint = checkNotNull(chainEntryPoint);
	this.headOperator = checkNotNull(headOperator);
}
 

@SafeVarargs
public static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(
		OneInputStreamOperator<T, T>... operators) throws Exception {

	checkNotNull(operators);
	checkArgument(operators.length > 0);

	try (MockEnvironment env = MockEnvironment.builder().build()) {
		final StreamTask<?, ?> containingTask = new MockStreamTaskBuilder(env).build();

		final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class);
		final StreamConfig cfg = new StreamConfig(new Configuration());

		final List<StreamOperatorWrapper<?, ?>> operatorWrappers = new ArrayList<>();

		// initial output goes to nowhere
		@SuppressWarnings({"unchecked", "rawtypes"})
		WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(
				new Output[0], statusProvider);

		// build the reverse operators array
		for (int i = 0; i < operators.length; i++) {
			OneInputStreamOperator<T, T> op = operators[operators.length - i - 1];
			if (op instanceof SetupableStreamOperator) {
				((SetupableStreamOperator) op).setup(containingTask, cfg, lastWriter);
			}
			lastWriter = new ChainingOutput<>(op, statusProvider, null);

			ProcessingTimeService processingTimeService = null;
			if (op instanceof AbstractStreamOperator) {
				processingTimeService = ((AbstractStreamOperator) op).getProcessingTimeService();
			}
			operatorWrappers.add(new StreamOperatorWrapper<>(
				op,
				Optional.ofNullable(processingTimeService),
				containingTask.getMailboxExecutorFactory().createExecutor(i)));
		}

		@SuppressWarnings("unchecked")
		final StreamOperatorWrapper<T, OP> headOperatorWrapper = (StreamOperatorWrapper<T, OP>) operatorWrappers.get(operatorWrappers.size() - 1);

		return new OperatorChain<>(
			operatorWrappers,
			new RecordWriterOutput<?>[0],
			lastWriter,
			headOperatorWrapper);
	}
}
 

public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
	for (RecordWriterOutput<?> streamOutput : streamOutputs) {
		streamOutput.broadcastEvent(event, isPriorityEvent);
	}
}
 

public RecordWriterOutput<?>[] getStreamOutputs() {
	return streamOutputs;
}
 

private <IN, OUT> WatermarkGaugeExposingOutput<StreamRecord<IN>> createChainedOperator(
		StreamTask<OUT, ?> containingTask,
		StreamConfig operatorConfig,
		Map<Integer, StreamConfig> chainedConfigs,
		ClassLoader userCodeClassloader,
		Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
		List<StreamOperatorWrapper<?, ?>> allOperatorWrappers,
		OutputTag<IN> outputTag,
		MailboxExecutorFactory mailboxExecutorFactory) {
	// create the output that the operator writes to first. this may recursively create more operators
	WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainedOperatorOutput = createOutputCollector(
		containingTask,
		operatorConfig,
		chainedConfigs,
		userCodeClassloader,
		streamOutputs,
		allOperatorWrappers,
		mailboxExecutorFactory);

	// now create the operator and give it the output collector to write its output to
	Tuple2<OneInputStreamOperator<IN, OUT>, Optional<ProcessingTimeService>> chainedOperatorAndTimeService =
		StreamOperatorFactoryUtil.createOperator(
			operatorConfig.getStreamOperatorFactory(userCodeClassloader),
			containingTask,
			operatorConfig,
			chainedOperatorOutput,
			operatorEventDispatcher);

	OneInputStreamOperator<IN, OUT> chainedOperator = chainedOperatorAndTimeService.f0;
	allOperatorWrappers.add(createOperatorWrapper(chainedOperator, containingTask, operatorConfig, chainedOperatorAndTimeService.f1));

	WatermarkGaugeExposingOutput<StreamRecord<IN>> currentOperatorOutput;
	if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
		currentOperatorOutput = new ChainingOutput<>(chainedOperator, this, outputTag);
	}
	else {
		TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader);
		currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
	}

	// wrap watermark gauges since registered metrics must be unique
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue);
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);

	return currentOperatorOutput;
}
 

RecordWriterOutput<?>[] getStreamOutputs() {
	return operatorChain.getStreamOutputs();
}
 

RecordWriterOutput<?>[] getStreamOutputs() {
	return operatorChain.getStreamOutputs();
}
 

@SafeVarargs
private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(
		OneInputStreamOperator<T, T>... operators) throws Exception {

	checkNotNull(operators);
	checkArgument(operators.length > 0);

	try (MockEnvironment env = MockEnvironment.builder().build()) {

	final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env);

		final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class);
		final StreamConfig cfg = new StreamConfig(new Configuration());

		final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length];

		// initial output goes to nowhere
		@SuppressWarnings({"unchecked", "rawtypes"})
		WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(
				new Output[0], statusProvider);

		// build the reverse operators array
		for (int i = 0; i < ops.length; i++) {
			OneInputStreamOperator<T, T> op = operators[ops.length - i - 1];
			if (op instanceof SetupableStreamOperator) {
				((SetupableStreamOperator) op).setup(containingTask, cfg, lastWriter);
			}
			lastWriter = new ChainingOutput<>(op, statusProvider, null);
			ops[i] = op;
		}

		@SuppressWarnings("unchecked")
		final OP head = (OP) operators[0];

		return new OperatorChain<>(
				ops,
				new RecordWriterOutput<?>[0],
				lastWriter,
				head);
	}
}
 

public RecordWriterOutput<?>[] getStreamOutputs() {
	return streamOutputs;
}
 

public void broadcastCheckpointCancelMarker(long id) throws IOException {
	CancelCheckpointMarker barrier = new CancelCheckpointMarker(id);
	for (RecordWriterOutput<?> streamOutput : streamOutputs) {
		streamOutput.broadcastEvent(barrier);
	}
}
 

public void broadcastCheckpointBarrier(long id, long timestamp, CheckpointOptions checkpointOptions) throws IOException {
	CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, checkpointOptions);
	for (RecordWriterOutput<?> streamOutput : streamOutputs) {
		streamOutput.broadcastEvent(barrier);
	}
}
 

RecordWriterOutput<?>[] getStreamOutputs() {
	return operatorChain.getStreamOutputs();
}
 

public RecordWriterOutput<?>[] getStreamOutputs() {
	return streamOutputs;
}
 

public void broadcastCheckpointCancelMarker(long id) throws IOException {
	CancelCheckpointMarker barrier = new CancelCheckpointMarker(id);
	for (RecordWriterOutput<?> streamOutput : streamOutputs) {
		streamOutput.broadcastEvent(barrier);
	}
}
 

public void broadcastCheckpointBarrier(long id, long timestamp, CheckpointOptions checkpointOptions) throws IOException {
	CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, checkpointOptions);
	for (RecordWriterOutput<?> streamOutput : streamOutputs) {
		streamOutput.broadcastEvent(barrier);
	}
}