org.apache.flink.streaming.api.transformations.CoFeedbackTransformation Java Examples

The following examples show how to use org.apache.flink.streaming.api.transformations.CoFeedbackTransformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IterativeStream.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public ConnectedIterativeStreams(DataStream<I> input,
		TypeInformation<F> feedbackType,
		long waitTime) {
	super(input.getExecutionEnvironment(),
			input,
			new DataStream<>(input.getExecutionEnvironment(),
					new CoFeedbackTransformation<>(input.getParallelism(),
							feedbackType,
							waitTime)));
	this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation();
}
Example #2
Source File: IterativeStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ConnectedIterativeStreams(DataStream<I> input,
		TypeInformation<F> feedbackType,
		long waitTime) {
	super(input.getExecutionEnvironment(),
			input,
			new DataStream<>(input.getExecutionEnvironment(),
					new CoFeedbackTransformation<>(input.getParallelism(),
							feedbackType,
							waitTime)));
	this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation();
}
Example #3
Source File: IterativeStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ConnectedIterativeStreams(DataStream<I> input,
		TypeInformation<F> feedbackType,
		long waitTime) {
	super(input.getExecutionEnvironment(),
			input,
			new DataStream<>(input.getExecutionEnvironment(),
					new CoFeedbackTransformation<>(input.getParallelism(),
							feedbackType,
							waitTime)));
	this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation();
}
Example #4
Source File: StreamGraphGenerator.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Transforms a {@code CoFeedbackTransformation}.
 *
 * <p>This will only transform feedback edges, the result of this transform will be wired
 * to the second input of a Co-Transform. The original input is wired directly to the first
 * input of the downstream Co-Transform.
 *
 * <p>This is responsible for creating the IterationSource and IterationSink which
 * are used to feed back the elements.
 */
private <F> Collection<Integer> transformCoFeedback(CoFeedbackTransformation<F> coIterate) {

	// For Co-Iteration we don't need to transform the input and wire the input to the
	// head operator by returning the input IDs, the input is directly wired to the left
	// input of the co-operation. This transform only needs to return the ids of the feedback
	// edges, since they need to be wired to the second input of the co-operation.

	// create the fake iteration source/sink pair
	Tuple2<StreamNode, StreamNode> itSourceAndSink = streamGraph.createIterationSourceAndSink(
			coIterate.getId(),
			getNewIterationNodeId(),
			getNewIterationNodeId(),
			coIterate.getWaitTime(),
			coIterate.getParallelism(),
			coIterate.getMaxParallelism(),
			coIterate.getMinResources(),
			coIterate.getPreferredResources());

	StreamNode itSource = itSourceAndSink.f0;
	StreamNode itSink = itSourceAndSink.f1;

	// We set the proper serializers for the sink/source
	streamGraph.setSerializers(itSource.getId(), null, null, coIterate.getOutputType().createSerializer(env.getConfig()));
	streamGraph.setSerializers(itSink.getId(), coIterate.getOutputType().createSerializer(env.getConfig()), null, null);

	Collection<Integer> resultIds = Collections.singleton(itSource.getId());

	// at the iterate to the already-seen-set with the result IDs, so that we can transform
	// the feedback edges and let them stop when encountering the iterate node
	alreadyTransformed.put(coIterate, resultIds);

	// so that we can determine the slot sharing group from all feedback edges
	List<Integer> allFeedbackIds = new ArrayList<>();

	for (StreamTransformation<F> feedbackEdge : coIterate.getFeedbackEdges()) {
		Collection<Integer> feedbackIds = transform(feedbackEdge);
		allFeedbackIds.addAll(feedbackIds);
		for (Integer feedbackId: feedbackIds) {
			streamGraph.addEdge(feedbackId,
					itSink.getId(),
					0
			);
		}
	}

	String slotSharingGroup = determineSlotSharingGroup(null, allFeedbackIds);

	itSink.setSlotSharingGroup(slotSharingGroup);
	itSource.setSlotSharingGroup(slotSharingGroup);

	return Collections.singleton(itSource.getId());
}
Example #5
Source File: StreamGraphGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Transforms a {@code CoFeedbackTransformation}.
 *
 * <p>This will only transform feedback edges, the result of this transform will be wired
 * to the second input of a Co-Transform. The original input is wired directly to the first
 * input of the downstream Co-Transform.
 *
 * <p>This is responsible for creating the IterationSource and IterationSink which
 * are used to feed back the elements.
 */
private <F> Collection<Integer> transformCoFeedback(CoFeedbackTransformation<F> coIterate) {

	// For Co-Iteration we don't need to transform the input and wire the input to the
	// head operator by returning the input IDs, the input is directly wired to the left
	// input of the co-operation. This transform only needs to return the ids of the feedback
	// edges, since they need to be wired to the second input of the co-operation.

	// create the fake iteration source/sink pair
	Tuple2<StreamNode, StreamNode> itSourceAndSink = streamGraph.createIterationSourceAndSink(
			coIterate.getId(),
			getNewIterationNodeId(),
			getNewIterationNodeId(),
			coIterate.getWaitTime(),
			coIterate.getParallelism(),
			coIterate.getMaxParallelism(),
			coIterate.getMinResources(),
			coIterate.getPreferredResources());

	StreamNode itSource = itSourceAndSink.f0;
	StreamNode itSink = itSourceAndSink.f1;

	// We set the proper serializers for the sink/source
	streamGraph.setSerializers(itSource.getId(), null, null, coIterate.getOutputType().createSerializer(executionConfig));
	streamGraph.setSerializers(itSink.getId(), coIterate.getOutputType().createSerializer(executionConfig), null, null);

	Collection<Integer> resultIds = Collections.singleton(itSource.getId());

	// at the iterate to the already-seen-set with the result IDs, so that we can transform
	// the feedback edges and let them stop when encountering the iterate node
	alreadyTransformed.put(coIterate, resultIds);

	// so that we can determine the slot sharing group from all feedback edges
	List<Integer> allFeedbackIds = new ArrayList<>();

	for (Transformation<F> feedbackEdge : coIterate.getFeedbackEdges()) {
		Collection<Integer> feedbackIds = transform(feedbackEdge);
		allFeedbackIds.addAll(feedbackIds);
		for (Integer feedbackId: feedbackIds) {
			streamGraph.addEdge(feedbackId,
					itSink.getId(),
					0
			);
		}
	}

	String slotSharingGroup = determineSlotSharingGroup(null, allFeedbackIds);

	itSink.setSlotSharingGroup(slotSharingGroup);
	itSource.setSlotSharingGroup(slotSharingGroup);

	return Collections.singleton(itSource.getId());
}
Example #6
Source File: StreamGraphGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Transforms a {@code CoFeedbackTransformation}.
 *
 * <p>This will only transform feedback edges, the result of this transform will be wired
 * to the second input of a Co-Transform. The original input is wired directly to the first
 * input of the downstream Co-Transform.
 *
 * <p>This is responsible for creating the IterationSource and IterationSink which
 * are used to feed back the elements.
 */
private <F> Collection<Integer> transformCoFeedback(CoFeedbackTransformation<F> coIterate) {

	// For Co-Iteration we don't need to transform the input and wire the input to the
	// head operator by returning the input IDs, the input is directly wired to the left
	// input of the co-operation. This transform only needs to return the ids of the feedback
	// edges, since they need to be wired to the second input of the co-operation.

	// create the fake iteration source/sink pair
	Tuple2<StreamNode, StreamNode> itSourceAndSink = streamGraph.createIterationSourceAndSink(
			coIterate.getId(),
			getNewIterationNodeId(),
			getNewIterationNodeId(),
			coIterate.getWaitTime(),
			coIterate.getParallelism(),
			coIterate.getMaxParallelism(),
			coIterate.getMinResources(),
			coIterate.getPreferredResources());

	StreamNode itSource = itSourceAndSink.f0;
	StreamNode itSink = itSourceAndSink.f1;

	// We set the proper serializers for the sink/source
	streamGraph.setSerializers(itSource.getId(), null, null, coIterate.getOutputType().createSerializer(executionConfig));
	streamGraph.setSerializers(itSink.getId(), coIterate.getOutputType().createSerializer(executionConfig), null, null);

	Collection<Integer> resultIds = Collections.singleton(itSource.getId());

	// at the iterate to the already-seen-set with the result IDs, so that we can transform
	// the feedback edges and let them stop when encountering the iterate node
	alreadyTransformed.put(coIterate, resultIds);

	// so that we can determine the slot sharing group from all feedback edges
	List<Integer> allFeedbackIds = new ArrayList<>();

	for (Transformation<F> feedbackEdge : coIterate.getFeedbackEdges()) {
		Collection<Integer> feedbackIds = transform(feedbackEdge);
		allFeedbackIds.addAll(feedbackIds);
		for (Integer feedbackId: feedbackIds) {
			streamGraph.addEdge(feedbackId,
					itSink.getId(),
					0
			);
		}
	}

	String slotSharingGroup = determineSlotSharingGroup(null, allFeedbackIds);

	itSink.setSlotSharingGroup(slotSharingGroup);
	itSource.setSlotSharingGroup(slotSharingGroup);

	return Collections.singleton(itSource.getId());
}