Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#getStreamGraph()

@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
 

/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 

@Test
public void testYieldingOperatorNotChainableToTaskChainedToLegacySource() {
	StreamExecutionEnvironment chainEnv = StreamExecutionEnvironment.createLocalEnvironment(1);

	chainEnv.fromElements(1)
		.map((x) -> x)
		// not chainable because of YieldingOperatorFactory and legacy source
		.transform("test", BasicTypeInfo.INT_TYPE_INFO, new YieldingTestOperatorFactory<>());

	final StreamGraph streamGraph = chainEnv.getStreamGraph();

	final List<StreamNode> streamNodes = streamGraph.getStreamNodes().stream()
		.sorted(Comparator.comparingInt(StreamNode::getId))
		.collect(Collectors.toList());
	assertTrue(areOperatorsChainable(streamNodes.get(0), streamNodes.get(1), streamGraph));
	assertFalse(areOperatorsChainable(streamNodes.get(1), streamNodes.get(2), streamGraph));
}
 

/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 

@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
 

/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 

@Override
public StreamGraph generateStreamGraph(List<Transformation<?>> transformations, String jobName) {
	StreamExecutionEnvironment execEnv = getExecutionEnvironment();
	setBatchProperties(execEnv);
	transformations.forEach(execEnv::addOperator);
	StreamGraph streamGraph;
	streamGraph = execEnv.getStreamGraph(getNonEmptyJobName(jobName));
	// All transformations should set managed memory size.
	ResourceSpec managedResourceSpec = NodeResourceUtil.fromManagedMem(0);
	streamGraph.getStreamNodes().forEach(sn -> {
		if (sn.getMinResources().equals(ResourceSpec.DEFAULT)) {
			sn.setResources(managedResourceSpec, managedResourceSpec);
		}
	});
	streamGraph.setChaining(true);
	streamGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST);
	streamGraph.setStateBackend(null);
	if (streamGraph.getCheckpointConfig().isCheckpointingEnabled()) {
		throw new IllegalArgumentException("Checkpoint is not supported for batch jobs.");
	}
	if (isShuffleModeAllBatch()) {
		streamGraph.setBlockingConnectionsBetweenChains(true);
	}
	return streamGraph;
}
 

/**
 * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the
 * client side.
 */
@Test
public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration);
}
 

/**
 * Create a StreamGraph as below.
 *
 * <p>source1 --(rebalance & pipelined)--> Map1
 *
 * <p>source2 --(rebalance & blocking)--> Map2
 */
private StreamGraph createStreamGraphForSlotSharingTest() {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStream<Integer> source1 = env.fromElements(1, 2, 3).name("source1");
	source1.rebalance().map(v -> v).name("map1");

	final DataStream<Integer> source2 = env.fromElements(4, 5, 6).name("source2");
	final DataStream<Integer> partitioned = new DataStream<>(env, new PartitionTransformation<>(
		source2.getTransformation(), new RebalancePartitioner<>(), ShuffleMode.BATCH));
	partitioned.map(v -> v).name("map2");

	return env.getStreamGraph();
}
 

@Test
public void testEnabledUnalignedCheckAndDisabledCheckpointing() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.fromElements(0).print();
	StreamGraph streamGraph = env.getStreamGraph();
	assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled());
	env.getCheckpointConfig().enableUnalignedCheckpoints(true);

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	StreamConfig streamConfig = new StreamConfig(verticesSorted.get(0).getConfiguration());
	assertEquals(CheckpointingMode.AT_LEAST_ONCE, streamConfig.getCheckpointMode());
	assertFalse(streamConfig.isUnalignedCheckpointsEnabled());
}
 

@Test
public void testDisablingAutoUidsAcceptsManuallySetHash() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.getConfig().disableAutoGeneratedUIDs();

	env
		.addSource(new NoOpSourceFunction()).setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
		.addSink(new DiscardingSink<>()).setUidHash("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");

	env.getStreamGraph();
}
 

private static StreamOperator<?> getOperatorForDataStream(DataStream<?> dataStream) {
	StreamExecutionEnvironment env = dataStream.getExecutionEnvironment();
	StreamGraph streamGraph = env.getStreamGraph();
	return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}
 

@Test
public void testBufferTimeout() {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	env.setBufferTimeout(77); // set timeout to some recognizable number

	env
		.fromElements(1, 2, 3, 4, 5)

		.map(value -> value)
			.setBufferTimeout(-1)
			.name("A")
		.map(value -> value)
			.setBufferTimeout(0)
			.name("B")
		.map(value -> value)
			.setBufferTimeout(12)
			.name("C")
		.map(value -> value)
			.name("D");

	final StreamGraph sg = env.getStreamGraph();
	for (StreamNode node : sg.getStreamNodes()) {
		switch (node.getOperatorName()) {

			case "A":
				assertEquals(77L, node.getBufferTimeout().longValue());
				break;
			case "B":
				assertEquals(0L, node.getBufferTimeout().longValue());
				break;
			case "C":
				assertEquals(12L, node.getBufferTimeout().longValue());
				break;
			case "D":
				assertEquals(77L, node.getBufferTimeout().longValue());
				break;
			default:
				assertTrue(node.getOperator() instanceof StreamSource);
		}
	}
}
 

private static StreamGraph dag(int numElements, boolean continueAfterNumElementsReached, int sinkDelayMillis, StreamExecutionEnvironment env) {
	env
		.addSource(CancellingIntegerSource.upTo(numElements, continueAfterNumElementsReached))
		.addSink(new AccumulatingIntegerSink(sinkDelayMillis));
	return env.getStreamGraph();
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

@Test
public void testBufferTimeout() {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	env.setBufferTimeout(77); // set timeout to some recognizable number

	env
		.fromElements(1, 2, 3, 4, 5)

		.map(value -> value)
			.setBufferTimeout(-1)
			.name("A")
		.map(value -> value)
			.setBufferTimeout(0)
			.name("B")
		.map(value -> value)
			.setBufferTimeout(12)
			.name("C")
		.map(value -> value)
			.name("D");

	final StreamGraph sg = env.getStreamGraph();
	for (StreamNode node : sg.getStreamNodes()) {
		switch (node.getOperatorName()) {

			case "A":
				assertEquals(77L, node.getBufferTimeout());
				break;
			case "B":
				assertEquals(0L, node.getBufferTimeout());
				break;
			case "C":
				assertEquals(12L, node.getBufferTimeout());
				break;
			case "D":
				assertEquals(77L, node.getBufferTimeout());
				break;
			default:
				assertTrue(node.getOperator() instanceof StreamSource);
		}
	}
}
 

@Test
public void testOutputTypeConfigurationWithTwoInputTransformation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source1 = env.fromElements(1, 10);
	DataStream<Integer> source2 = env.fromElements(2, 11);

	ConnectedStreams<Integer, Integer> connectedSource = source1.connect(source2);

	OutputTypeConfigurableOperationWithTwoInputs outputTypeConfigurableOperation = new OutputTypeConfigurableOperationWithTwoInputs();

	DataStream<Integer> result = connectedSource.transform(
			"Two input and output type configurable operation",
			BasicTypeInfo.INT_TYPE_INFO,
			outputTypeConfigurableOperation);

	result.addSink(new DiscardingSink<>());

	env.getStreamGraph();

	assertEquals(BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation.getTypeInformation());
}
 

/**
 * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of
 * maximum parallelism.
 */
@Test
public void testSetupOfKeyGroupPartitioner() {
	int maxParallelism = 42;
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().setMaxParallelism(maxParallelism);

	DataStream<Integer> source = env.fromElements(1, 2, 3);

	DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap());

	keyedResult.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());

	StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner();
}
 

@Test
public void testInputOutputFormat() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Long> source = env.addSource(
		new InputFormatSourceFunction<>(
			new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)),
			TypeInformation.of(Long.class)),
		TypeInformation.of(Long.class)).name("source");

	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");

	StreamGraph streamGraph = env.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	assertEquals(1, jobGraph.getNumberOfVertices());

	JobVertex jobVertex = jobGraph.getVertices().iterator().next();
	assertTrue(jobVertex instanceof InputOutputFormatVertex);

	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(
		new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
	Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(2, outputFormats.size());

	Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
	StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
	nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());

	Map<Integer, StreamConfig> chainedConfigs = headConfig
		.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
	for (StreamConfig config : chainedConfigs.values()) {
		nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
	}

	InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
	assertTrue(sourceFormat instanceof TypeSerializerInputFormat);

	OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
	assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);

	OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
	assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}
 

@Test
public void testKeybyBetweenConsecutiveSplitRejection() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Integer> src = env.fromElements(0, 0);

	OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();

	src.split(outputSelector).select("dummy").keyBy(x -> x).split(outputSelector).addSink(new DiscardingSink<>());

	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs.");

	env.getStreamGraph();
}