Java Code Examples for org.apache.flink.runtime.jobgraph.JobVertex#getConfiguration()

@Test
public void testManagedMemoryFractionForUnknownResourceSpec() throws Exception {
	final ResourceSpec resource = ResourceSpec.UNKNOWN;
	final List<ResourceSpec> resourceSpecs = Arrays.asList(resource, resource, resource, resource);
	final List<Integer> managedMemoryWeights = Arrays.asList(1, 2, 3, 4);

	// v1(source -> map1), v2(map2) are in the same slot sharing group, v3(map3) is in a different group
	final JobGraph jobGraph = createJobGraphForManagedMemoryFractionTest(resourceSpecs, managedMemoryWeights);
	final JobVertex vertex1 = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
	final JobVertex vertex2 = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	final JobVertex vertex3 = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);

	final StreamConfig sourceConfig = new StreamConfig(vertex1.getConfiguration());
	assertEquals(1.0 / 6, sourceConfig.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map1Config = Iterables.getOnlyElement(
		sourceConfig.getTransitiveChainedTaskConfigs(StreamingJobGraphGeneratorTest.class.getClassLoader()).values());
	assertEquals(2.0 / 6, map1Config.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map2Config = new StreamConfig(vertex2.getConfiguration());
	assertEquals(3.0 / 6, map2Config.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map3Config = new StreamConfig(vertex3.getConfiguration());
	assertEquals(1.0, map3Config.getManagedMemoryFraction(), 0.000001);

}
 

private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
	final String taskName = node.getNodeName();
	final DriverStrategy ds = node.getDriverStrategy();
	final JobVertex vertex = new JobVertex(taskName);
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());
	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class);
	
	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());
	
	// set the driver strategy
	config.setDriver(ds.getDriverClass());
	config.setDriverStrategy(ds);
	if (node.getComparator1() != null) {
		config.setDriverComparator(node.getComparator1(), 0);
	}
	if (node.getComparator2() != null) {
		config.setDriverComparator(node.getComparator2(), 1);
	}
	if (node.getPairComparator() != null) {
		config.setDriverPairComparator(node.getPairComparator());
	}
	
	// assign memory, file-handles, etc.
	assignDriverResources(node, config);
	return vertex;
}
 

@Test
public void testTempInIterationTest() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class);

	DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
			input.iterateDelta(input, 1, 0);

	DataSet<Tuple2<Long, Long>> update = iteration.getWorkset()
			.join(iteration.getSolutionSet()).where(0).equalTo(0)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

	iteration.closeWith(update, update)
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());


	Plan plan = env.createProgramPlan();
	OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jg = jgg.compileJobGraph(oPlan);

	boolean solutionSetUpdateChecked = false;
	for(JobVertex v : jg.getVertices()) {
		if(v.getName().equals("SolutionSet Delta")) {

			// check if input of solution set delta is temped
			TaskConfig tc = new TaskConfig(v.getConfiguration());
			assertTrue(tc.isInputAsynchronouslyMaterialized(0));
			solutionSetUpdateChecked = true;
		}
	}
	assertTrue(solutionSetUpdateChecked);

}
 

private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
	final String taskName = node.getNodeName();
	final DriverStrategy ds = node.getDriverStrategy();
	final JobVertex vertex = new JobVertex(taskName);
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());
	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class);
	
	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());
	
	// set the driver strategy
	config.setDriver(ds.getDriverClass());
	config.setDriverStrategy(ds);
	if (node.getComparator1() != null) {
		config.setDriverComparator(node.getComparator1(), 0);
	}
	if (node.getComparator2() != null) {
		config.setDriverComparator(node.getComparator2(), 1);
	}
	if (node.getPairComparator() != null) {
		config.setDriverPairComparator(node.getPairComparator());
	}
	
	// assign memory, file-handles, etc.
	assignDriverResources(node, config);
	return vertex;
}
 

/**
 * Verifies that the chain start/end is correctly set.
 */
@Test
public void testChainStartEndSetting() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// set parallelism to 2 to avoid chaining with source in case when available processors is 1.
	env.setParallelism(2);

	// fromElements -> CHAIN(Map -> Print)
	env.fromElements(1, 2, 3)
		.map(new MapFunction<Integer, Integer>() {
			@Override
			public Integer map(Integer value) throws Exception {
				return value;
			}
		})
		.print();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	JobVertex sourceVertex = verticesSorted.get(0);
	JobVertex mapPrintVertex = verticesSorted.get(1);

	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());

	StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
	StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
	Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
	StreamConfig printConfig = chainedConfigs.values().iterator().next();

	assertTrue(sourceConfig.isChainStart());
	assertTrue(sourceConfig.isChainEnd());

	assertTrue(mapConfig.isChainStart());
	assertFalse(mapConfig.isChainEnd());

	assertFalse(printConfig.isChainStart());
	assertTrue(printConfig.isChainEnd());
}
 

@Test
public void testTempInIterationTest() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class);

	DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
			input.iterateDelta(input, 1, 0);

	DataSet<Tuple2<Long, Long>> update = iteration.getWorkset()
			.join(iteration.getSolutionSet()).where(0).equalTo(0)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

	iteration.closeWith(update, update)
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());


	Plan plan = env.createProgramPlan();
	OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jg = jgg.compileJobGraph(oPlan);

	boolean solutionSetUpdateChecked = false;
	for(JobVertex v : jg.getVertices()) {
		if(v.getName().equals("SolutionSet Delta")) {

			// check if input of solution set delta is temped
			TaskConfig tc = new TaskConfig(v.getConfiguration());
			assertTrue(tc.isInputAsynchronouslyMaterialized(0));
			solutionSetUpdateChecked = true;
		}
	}
	assertTrue(solutionSetUpdateChecked);

}
 

private void connect(Integer headOfChain, StreamEdge edge) {

		physicalEdgesInOrder.add(edge);

		Integer downStreamvertexID = edge.getTargetId();

		JobVertex headVertex = jobVertices.get(headOfChain);
		JobVertex downStreamVertex = jobVertices.get(downStreamvertexID);

		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());

		downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1);

		StreamPartitioner<?> partitioner = edge.getPartitioner();
		JobEdge jobEdge;
		if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
				headVertex,
				DistributionPattern.POINTWISE,
				ResultPartitionType.PIPELINED_BOUNDED);
		} else {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
					headVertex,
					DistributionPattern.ALL_TO_ALL,
					ResultPartitionType.PIPELINED_BOUNDED);
		}
		// set strategy name so that web interface can show it.
		jobEdge.setShipStrategyName(partitioner.toString());

		if (LOG.isDebugEnabled()) {
			LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
					headOfChain, downStreamvertexID);
		}
	}
 

/**
 * Verifies that the chain start/end is correctly set.
 */
@Test
public void testChainStartEndSetting() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// set parallelism to 2 to avoid chaining with source in case when available processors is 1.
	env.setParallelism(2);

	// fromElements -> CHAIN(Map -> Print)
	env.fromElements(1, 2, 3)
		.map(new MapFunction<Integer, Integer>() {
			@Override
			public Integer map(Integer value) throws Exception {
				return value;
			}
		})
		.print();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	JobVertex sourceVertex = verticesSorted.get(0);
	JobVertex mapPrintVertex = verticesSorted.get(1);

	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());

	StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
	StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
	Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
	StreamConfig printConfig = chainedConfigs.values().iterator().next();

	assertTrue(sourceConfig.isChainStart());
	assertTrue(sourceConfig.isChainEnd());

	assertTrue(mapConfig.isChainStart());
	assertFalse(mapConfig.isChainEnd());

	assertFalse(printConfig.isChainStart());
	assertTrue(printConfig.isChainEnd());
}
 

@Test
public void testTempInIterationTest() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class);

	DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
			input.iterateDelta(input, 1, 0);

	DataSet<Tuple2<Long, Long>> update = iteration.getWorkset()
			.join(iteration.getSolutionSet()).where(0).equalTo(0)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

	iteration.closeWith(update, update)
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());


	Plan plan = env.createProgramPlan();
	OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jg = jgg.compileJobGraph(oPlan);

	boolean solutionSetUpdateChecked = false;
	for(JobVertex v : jg.getVertices()) {
		if(v.getName().equals("SolutionSet Delta")) {

			// check if input of solution set delta is temped
			TaskConfig tc = new TaskConfig(v.getConfiguration());
			assertTrue(tc.isInputAsynchronouslyMaterialized(0));
			solutionSetUpdateChecked = true;
		}
	}
	assertTrue(solutionSetUpdateChecked);

}
 

private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
	final String taskName = node.getNodeName();
	final DriverStrategy ds = node.getDriverStrategy();
	final JobVertex vertex = new JobVertex(taskName);
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());
	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class);
	
	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());
	
	// set the driver strategy
	config.setDriver(ds.getDriverClass());
	config.setDriverStrategy(ds);
	if (node.getComparator1() != null) {
		config.setDriverComparator(node.getComparator1(), 0);
	}
	if (node.getComparator2() != null) {
		config.setDriverComparator(node.getComparator2(), 1);
	}
	if (node.getPairComparator() != null) {
		config.setDriverPairComparator(node.getPairComparator());
	}
	
	// assign memory, file-handles, etc.
	assignDriverResources(node, config);
	return vertex;
}
 

/**
 * Verifies that the chain start/end is correctly set.
 */
@Test
public void testChainStartEndSetting() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	// fromElements -> CHAIN(Map -> Print)
	env.fromElements(1, 2, 3)
		.map(new MapFunction<Integer, Integer>() {
			@Override
			public Integer map(Integer value) throws Exception {
				return value;
			}
		})
		.print();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	JobVertex sourceVertex = verticesSorted.get(0);
	JobVertex mapPrintVertex = verticesSorted.get(1);

	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());

	StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
	StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
	Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
	StreamConfig printConfig = chainedConfigs.values().iterator().next();

	assertTrue(sourceConfig.isChainStart());
	assertTrue(sourceConfig.isChainEnd());

	assertTrue(mapConfig.isChainStart());
	assertFalse(mapConfig.isChainEnd());

	assertFalse(printConfig.isChainStart());
	assertTrue(printConfig.isChainEnd());
}
 

@Test
public void testInputOutputFormat() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Long> source = env.addSource(
		new InputFormatSourceFunction<>(
			new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)),
			TypeInformation.of(Long.class)),
		TypeInformation.of(Long.class)).name("source");

	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");

	StreamGraph streamGraph = env.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	assertEquals(1, jobGraph.getNumberOfVertices());

	JobVertex jobVertex = jobGraph.getVertices().iterator().next();
	assertTrue(jobVertex instanceof InputOutputFormatVertex);

	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(
		new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
	Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(2, outputFormats.size());

	Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
	StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
	nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());

	Map<Integer, StreamConfig> chainedConfigs = headConfig
		.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
	for (StreamConfig config : chainedConfigs.values()) {
		nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
	}

	InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
	assertTrue(sourceFormat instanceof TypeSerializerInputFormat);

	OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
	assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);

	OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
	assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}
 

/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(false);

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	final AsyncWaitOperator<Integer, Integer> headOperator =
			operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
	streamConfig.setStreamOperator(headOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}
 

private void connect(Integer headOfChain, StreamEdge edge) {

		physicalEdgesInOrder.add(edge);

		Integer downStreamvertexID = edge.getTargetId();

		JobVertex headVertex = jobVertices.get(headOfChain);
		JobVertex downStreamVertex = jobVertices.get(downStreamvertexID);

		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());

		downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1);

		StreamPartitioner<?> partitioner = edge.getPartitioner();

		ResultPartitionType resultPartitionType;
		switch (edge.getShuffleMode()) {
			case PIPELINED:
				resultPartitionType = ResultPartitionType.PIPELINED_BOUNDED;
				break;
			case BATCH:
				resultPartitionType = ResultPartitionType.BLOCKING;
				break;
			case UNDEFINED:
				resultPartitionType = streamGraph.isBlockingConnectionsBetweenChains() ?
						ResultPartitionType.BLOCKING : ResultPartitionType.PIPELINED_BOUNDED;
				break;
			default:
				throw new UnsupportedOperationException("Data exchange mode " +
					edge.getShuffleMode() + " is not supported yet.");
		}

		JobEdge jobEdge;
		if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
				headVertex,
				DistributionPattern.POINTWISE,
				resultPartitionType);
		} else {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
					headVertex,
					DistributionPattern.ALL_TO_ALL,
					resultPartitionType);
		}
		// set strategy name so that web interface can show it.
		jobEdge.setShipStrategyName(partitioner.toString());

		if (LOG.isDebugEnabled()) {
			LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
					headOfChain, downStreamvertexID);
		}
	}
 

private void connect(Integer headOfChain, StreamEdge edge) {

		physicalEdgesInOrder.add(edge);

		Integer downStreamVertexID = edge.getTargetId();

		JobVertex headVertex = jobVertices.get(headOfChain);
		JobVertex downStreamVertex = jobVertices.get(downStreamVertexID);

		StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration());

		downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1);

		StreamPartitioner<?> partitioner = edge.getPartitioner();

		ResultPartitionType resultPartitionType;
		switch (edge.getShuffleMode()) {
			case PIPELINED:
				resultPartitionType = ResultPartitionType.PIPELINED_BOUNDED;
				break;
			case BATCH:
				resultPartitionType = ResultPartitionType.BLOCKING;
				break;
			case UNDEFINED:
				resultPartitionType = determineResultPartitionType(partitioner);
				break;
			default:
				throw new UnsupportedOperationException("Data exchange mode " +
					edge.getShuffleMode() + " is not supported yet.");
		}

		JobEdge jobEdge;
		if (isPointwisePartitioner(partitioner)) {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
				headVertex,
				DistributionPattern.POINTWISE,
				resultPartitionType);
		} else {
			jobEdge = downStreamVertex.connectNewDataSetAsInput(
					headVertex,
					DistributionPattern.ALL_TO_ALL,
					resultPartitionType);
		}
		// set strategy name so that web interface can show it.
		jobEdge.setShipStrategyName(partitioner.toString());

		if (LOG.isDebugEnabled()) {
			LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(),
					headOfChain, downStreamVertexID);
		}
	}
 

/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(new MyAsyncFunction(), new MyAsyncFunction());

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	streamConfig.setStreamOperatorFactory(
			operatorChainStreamConfig.getStreamOperatorFactory(AsyncWaitOperatorTest.class.getClassLoader()));

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	List<Object> expectedOutput = new LinkedList<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}
 

@Test
public void testInputOutputFormat() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Long> source = env.addSource(
		new InputFormatSourceFunction<>(
			new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)),
			TypeInformation.of(Long.class)),
		TypeInformation.of(Long.class)).name("source");

	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
	source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");

	StreamGraph streamGraph = env.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	assertEquals(1, jobGraph.getNumberOfVertices());

	JobVertex jobVertex = jobGraph.getVertices().iterator().next();
	assertTrue(jobVertex instanceof InputOutputFormatVertex);

	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(
		new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
	Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(2, outputFormats.size());

	Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
	StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
	nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());

	Map<Integer, StreamConfig> chainedConfigs = headConfig
		.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
	for (StreamConfig config : chainedConfigs.values()) {
		nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
	}

	InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
	assertTrue(sourceFormat instanceof TypeSerializerInputFormat);

	OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
	assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);

	OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
	assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}
 

@Test
public void testManagedMemoryFractionForSpecifiedResourceSpec() throws Exception {
	// these specific values are needed to produce the double precision issue,
	// i.e. 100.0 / 1100 + 300.0 / 1100 + 700.0 / 1100 can be larger than 1.0.
	final ResourceSpec resource1 = ResourceSpec.newBuilder(1, 100)
		.setManagedMemory(new MemorySize(100))
		.build();
	final ResourceSpec resource2 = ResourceSpec.newBuilder(1, 100)
		.setManagedMemory(new MemorySize(300))
		.build();
	final ResourceSpec resource3 = ResourceSpec.newBuilder(1, 100)
		.setManagedMemory(new MemorySize(700))
		.build();
	final ResourceSpec resource4 = ResourceSpec.newBuilder(1, 100)
		.setManagedMemory(new MemorySize(123))
		.build();
	final List<ResourceSpec> resourceSpecs = Arrays.asList(resource1, resource2, resource3, resource4);

	// v1(source -> map1), v2(map2) are in the same slot sharing group, v3(map3) is in a different group
	final JobGraph jobGraph = createJobGraphForManagedMemoryFractionTest(resourceSpecs, null);
	final JobVertex vertex1 = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
	final JobVertex vertex2 = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	final JobVertex vertex3 = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);

	final StreamConfig sourceConfig = new StreamConfig(vertex1.getConfiguration());
	assertEquals(100.0 / 1100, sourceConfig.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map1Config = Iterables.getOnlyElement(
		sourceConfig.getTransitiveChainedTaskConfigs(StreamingJobGraphGeneratorTest.class.getClassLoader()).values());
	assertEquals(300.0 / 1100, map1Config.getManagedMemoryFraction(), 0.000001);

	final StreamConfig map2Config = new StreamConfig(vertex2.getConfiguration());
	assertEquals(700.0 / 1100, map2Config.getManagedMemoryFraction(), 0.000001);

	final BigDecimal sumFraction = BigDecimal.valueOf(sourceConfig.getManagedMemoryFraction())
		.add(BigDecimal.valueOf(map1Config.getManagedMemoryFraction()))
		.add(BigDecimal.valueOf(map2Config.getManagedMemoryFraction()));
	assertThat(sumFraction, lessThanOrEqualTo(BigDecimal.ONE));

	final StreamConfig map3Config = new StreamConfig(vertex3.getConfiguration());
	assertEquals(1.0, map3Config.getManagedMemoryFraction(), 0.000001);
}
 

/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(false);

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	final AsyncWaitOperator<Integer, Integer> headOperator =
			operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
	streamConfig.setStreamOperator(headOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}