org.apache.flink.streaming.api.transformations.SinkTransformation Java Examples

The following examples show how to use org.apache.flink.streaming.api.transformations.SinkTransformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkKafkaShuffle.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Adds a {@link StreamKafkaShuffleSink} to {@link DataStream}.
 *
 * <p>{@link StreamKafkaShuffleSink} is associated a {@link FlinkKafkaShuffleProducer}.
 *
 * @param inputStream 				Input data stream connected to the shuffle
 * @param kafkaShuffleProducer 		Kafka shuffle sink function that can handle both records and watermark
 * @param producerParallelism 		The number of tasks writing to the kafka shuffle
 */
private static <T, K> void addKafkaShuffle(
		DataStream<T> inputStream,
		FlinkKafkaShuffleProducer<T, K> kafkaShuffleProducer,
		int producerParallelism) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	inputStream.getTransformation().getOutputType();

	StreamKafkaShuffleSink<T> shuffleSinkOperator = new StreamKafkaShuffleSink<>(kafkaShuffleProducer);
	SinkTransformation<T> transformation = new SinkTransformation<>(
		inputStream.getTransformation(),
		"kafka_shuffle",
		shuffleSinkOperator,
		inputStream.getExecutionEnvironment().getParallelism());
	inputStream.getExecutionEnvironment().addOperator(transformation);
	transformation.setParallelism(producerParallelism);
}
Example #2
Source File: StreamGraphGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Transforms a {@code SourceTransformation}.
 */
private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {

	Collection<Integer> inputIds = transform(sink.getInput());

	String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds);

	streamGraph.addSink(sink.getId(),
			slotSharingGroup,
			sink.getCoLocationGroupKey(),
			sink.getOperatorFactory(),
			sink.getInput().getOutputType(),
			null,
			"Sink: " + sink.getName());

	StreamOperatorFactory operatorFactory = sink.getOperatorFactory();
	if (operatorFactory instanceof OutputFormatOperatorFactory) {
		streamGraph.setOutputFormat(sink.getId(), ((OutputFormatOperatorFactory) operatorFactory).getOutputFormat());
	}

	int parallelism = sink.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		sink.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(sink.getId(), parallelism);
	streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId,
				sink.getId(),
				0
		);
	}

	if (sink.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer);
	}

	return Collections.emptyList();
}
Example #3
Source File: StreamGraphGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Transforms a {@code SinkTransformation}.
 */
private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {

	Collection<Integer> inputIds = transform(sink.getInput());

	String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds);

	streamGraph.addSink(sink.getId(),
			slotSharingGroup,
			sink.getCoLocationGroupKey(),
			sink.getOperatorFactory(),
			sink.getInput().getOutputType(),
			null,
			"Sink: " + sink.getName());

	StreamOperatorFactory operatorFactory = sink.getOperatorFactory();
	if (operatorFactory instanceof OutputFormatOperatorFactory) {
		streamGraph.setOutputFormat(sink.getId(), ((OutputFormatOperatorFactory) operatorFactory).getOutputFormat());
	}

	int parallelism = sink.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		sink.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(sink.getId(), parallelism);
	streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId,
				sink.getId(),
				0
		);
	}

	if (sink.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer);
	}

	return Collections.emptyList();
}
Example #4
Source File: CollectStreamSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
public CollectStreamSink(DataStream<T> inputStream, CollectSinkOperatorFactory<T> factory) {
	super(inputStream, (CollectSinkOperator<T>) factory.getOperator());
	this.transformation = new SinkTransformation<>(
		inputStream.getTransformation(),
		"Collect Stream Sink",
		factory,
		1);
}
Example #5
Source File: StreamGraphGenerator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Transforms a {@code SourceTransformation}.
 */
private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {

	Collection<Integer> inputIds = transform(sink.getInput());

	String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds);

	streamGraph.addSink(sink.getId(),
			slotSharingGroup,
			sink.getCoLocationGroupKey(),
			sink.getOperator(),
			sink.getInput().getOutputType(),
			null,
			"Sink: " + sink.getName());

	streamGraph.setParallelism(sink.getId(), sink.getParallelism());
	streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId,
				sink.getId(),
				0
		);
	}

	if (sink.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(env.getConfig());
		streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer);
	}

	return Collections.emptyList();
}
Example #6
Source File: CassandraSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
private SinkTransformation<IN> getSinkTransformation() {
	return sink1.getTransformation();
}
Example #7
Source File: FlinkKafkaProducer010.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #8
Source File: DataStreamSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected DataStreamSink(DataStream<T> inputStream, StreamSink<T> operator) {
	this.transformation = new SinkTransformation<T>(inputStream.getTransformation(), "Unnamed", operator, inputStream.getExecutionEnvironment().getParallelism());
}
Example #9
Source File: DataStreamSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Returns the transformation that contains the actual sink operator of this sink.
 */
@Internal
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #10
Source File: CassandraSink.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private SinkTransformation<IN> getSinkTransformation() {
	return sink1.getTransformation();
}
Example #11
Source File: CassandraSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
private SinkTransformation<IN> getSinkTransformation() {
	return sink1.getTransformation();
}
Example #12
Source File: DataStreamSink.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Returns the transformation that contains the actual sink operator of this sink.
 */
@Internal
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #13
Source File: FlinkKafkaProducer010.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #14
Source File: DataStreamSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected DataStreamSink(DataStream<T> inputStream, StreamSink<T> operator) {
	this.transformation = new SinkTransformation<T>(inputStream.getTransformation(), "Unnamed", operator, inputStream.getExecutionEnvironment().getParallelism());
}
Example #15
Source File: DataStreamSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Returns the transformation that contains the actual sink operator of this sink.
 */
@Internal
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #16
Source File: DataStreamSink.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected DataStreamSink(DataStream<T> inputStream, StreamSink<T> operator) {
	this.transformation = new SinkTransformation<T>(inputStream.getTransformation(), "Unnamed", operator, inputStream.getExecutionEnvironment().getParallelism());
}
Example #17
Source File: CollectStreamSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public SinkTransformation<T> getTransformation() {
	return transformation;
}
Example #18
Source File: FlinkKafkaProducer010.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public SinkTransformation<T> getTransformation() {
	return transformation;
}