org.apache.flink.streaming.runtime.tasks.StreamIterationHead Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.tasks.StreamIterationHead. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public Tuple2<StreamNode, StreamNode> createIterationSourceAndSink(
	int loopId,
	int sourceId,
	int sinkId,
	long timeout,
	int parallelism,
	int maxParallelism,
	ResourceSpec minResources,
	ResourceSpec preferredResources) {
	StreamNode source = this.addNode(sourceId,
		null,
		null,
		StreamIterationHead.class,
		null,
		"IterationSource-" + loopId);
	sources.add(source.getId());
	setParallelism(source.getId(), parallelism);
	setMaxParallelism(source.getId(), maxParallelism);
	setResources(source.getId(), minResources, preferredResources);

	StreamNode sink = this.addNode(sinkId,
		null,
		null,
		StreamIterationTail.class,
		null,
		"IterationSink-" + loopId);
	sinks.add(sink.getId());
	setParallelism(sink.getId(), parallelism);
	setMaxParallelism(sink.getId(), parallelism);

	iterationSourceSinkPairs.add(new Tuple2<>(source, sink));

	this.vertexIDtoBrokerID.put(source.getId(), "broker-" + loopId);
	this.vertexIDtoBrokerID.put(sink.getId(), "broker-" + loopId);
	this.vertexIDtoLoopTimeout.put(source.getId(), timeout);
	this.vertexIDtoLoopTimeout.put(sink.getId(), timeout);

	return new Tuple2<>(source, sink);
}
Example #2
Source File: StreamIterationHeadTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testIterationHeadWatermarkEmission() throws Exception {
	StreamTaskTestHarness<Integer> harness = new StreamTaskTestHarness<>(
			StreamIterationHead::new,
			BasicTypeInfo.INT_TYPE_INFO);
	harness.setupOutputForSingletonOperatorChain();
	harness.getStreamConfig().setIterationId("1");
	harness.getStreamConfig().setIterationWaitTime(1);

	harness.invoke();
	harness.waitForTaskCompletion();

	assertEquals(1, harness.getOutput().size());
	assertEquals(new Watermark(Long.MAX_VALUE), harness.getOutput().peek());
}
Example #3
Source File: StreamIterationHeadTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testIterationHeadWatermarkEmission() throws Exception {
	StreamTaskTestHarness<Integer> harness = new StreamTaskTestHarness<>(
			StreamIterationHead::new,
			BasicTypeInfo.INT_TYPE_INFO);
	harness.setupOutputForSingletonOperatorChain();
	harness.getStreamConfig().setIterationId("1");
	harness.getStreamConfig().setIterationWaitTime(1);

	harness.invoke();
	harness.waitForTaskCompletion();

	assertEquals(1, harness.getOutput().size());
	assertEquals(new Watermark(Long.MAX_VALUE), harness.getOutput().peek());
}
Example #4
Source File: StreamIterationHeadTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testIterationHeadWatermarkEmission() throws Exception {
	StreamTaskTestHarness<Integer> harness = new StreamTaskTestHarness<>(
			StreamIterationHead::new,
			BasicTypeInfo.INT_TYPE_INFO);
	harness.setupOutputForSingletonOperatorChain();
	harness.getStreamConfig().setIterationId("1");
	harness.getStreamConfig().setIterationWaitTime(1);

	harness.invoke();
	harness.waitForTaskCompletion();

	assertEquals(1, harness.getOutput().size());
	assertEquals(new Watermark(Long.MAX_VALUE), harness.getOutput().peek());
}
Example #5
Source File: StreamGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public Tuple2<StreamNode, StreamNode> createIterationSourceAndSink(
	int loopId,
	int sourceId,
	int sinkId,
	long timeout,
	int parallelism,
	int maxParallelism,
	ResourceSpec minResources,
	ResourceSpec preferredResources) {

	final String coLocationGroup = "IterationCoLocationGroup-" + loopId;

	StreamNode source = this.addNode(sourceId,
		null,
		coLocationGroup,
		StreamIterationHead.class,
		null,
		ITERATION_SOURCE_NAME_PREFIX + "-" + loopId);
	sources.add(source.getId());
	setParallelism(source.getId(), parallelism);
	setMaxParallelism(source.getId(), maxParallelism);
	setResources(source.getId(), minResources, preferredResources);

	StreamNode sink = this.addNode(sinkId,
		null,
		coLocationGroup,
		StreamIterationTail.class,
		null,
		ITERATION_SINK_NAME_PREFIX + "-" + loopId);
	sinks.add(sink.getId());
	setParallelism(sink.getId(), parallelism);
	setMaxParallelism(sink.getId(), parallelism);

	iterationSourceSinkPairs.add(new Tuple2<>(source, sink));

	this.vertexIDtoBrokerID.put(source.getId(), "broker-" + loopId);
	this.vertexIDtoBrokerID.put(sink.getId(), "broker-" + loopId);
	this.vertexIDtoLoopTimeout.put(source.getId(), timeout);
	this.vertexIDtoLoopTimeout.put(sink.getId(), timeout);

	return new Tuple2<>(source, sink);
}
Example #6
Source File: StreamGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public Tuple2<StreamNode, StreamNode> createIterationSourceAndSink(
	int loopId,
	int sourceId,
	int sinkId,
	long timeout,
	int parallelism,
	int maxParallelism,
	ResourceSpec minResources,
	ResourceSpec preferredResources) {

	final String coLocationGroup = "IterationCoLocationGroup-" + loopId;

	StreamNode source = this.addNode(sourceId,
		null,
		coLocationGroup,
		StreamIterationHead.class,
		null,
		ITERATION_SOURCE_NAME_PREFIX + "-" + loopId);
	sources.add(source.getId());
	setParallelism(source.getId(), parallelism);
	setMaxParallelism(source.getId(), maxParallelism);
	setResources(source.getId(), minResources, preferredResources);

	StreamNode sink = this.addNode(sinkId,
		null,
		coLocationGroup,
		StreamIterationTail.class,
		null,
		ITERATION_SINK_NAME_PREFIX + "-" + loopId);
	sinks.add(sink.getId());
	setParallelism(sink.getId(), parallelism);
	setMaxParallelism(sink.getId(), parallelism);
	// The tail node is always in the same slot sharing group with the head node
	// so that they can share resources (they do not use non-sharable resources,
	// i.e. managed memory). There is no contract on how the resources should be
	// divided for head and tail nodes at the moment. To be simple, we assign all
	// resources to the head node and set the tail node resources to be zero if
	// resources are specified.
	final ResourceSpec tailResources = minResources.equals(ResourceSpec.UNKNOWN)
		? ResourceSpec.UNKNOWN
		: ResourceSpec.ZERO;
	setResources(sink.getId(), tailResources, tailResources);

	iterationSourceSinkPairs.add(new Tuple2<>(source, sink));

	this.vertexIDtoBrokerID.put(source.getId(), "broker-" + loopId);
	this.vertexIDtoBrokerID.put(sink.getId(), "broker-" + loopId);
	this.vertexIDtoLoopTimeout.put(source.getId(), timeout);
	this.vertexIDtoLoopTimeout.put(sink.getId(), timeout);

	return new Tuple2<>(source, sink);
}