org.apache.flink.runtime.jobgraph.JobVertex Java Examples

@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

private static JobGraph createJobGraph() {
	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(TestSourceInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(group1);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(TestSinkInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(group2);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES);

	return jobGraph;
}
 

public static ExecutionJobVertex getExecutionVertex(
		JobVertexID id,
		ScheduledExecutorService executor,
		ScheduleMode scheduleMode) throws Exception {

	JobVertex ajv = new JobVertex("TestVertex", id);
	ajv.setInvokableClass(AbstractInvokable.class);

	ExecutionGraph graph = new TestingExecutionGraphBuilder(ajv)
		.setIoExecutor(executor)
		.setFutureExecutor(executor)
		.setScheduleMode(scheduleMode)
		.build();

	graph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	return new ExecutionJobVertex(graph, ajv, 1, AkkaUtils.getDefaultTimeout());
}
 

public static ExecutionJobVertex getExecutionVertex(
		JobVertexID id, ScheduledExecutorService executor) 
	throws Exception {

	JobVertex ajv = new JobVertex("TestVertex", id);
	ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());

	ExecutionGraph graph = new ExecutionGraph(
		executor,
		executor,
		new JobID(), 
		"test job", 
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()));

	graph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	return spy(new ExecutionJobVertex(graph, ajv, 1, AkkaUtils.getDefaultTimeout()));
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}
 

/**
 * Tests that there are no collisions with two identical sources.
 *
 * <pre>
 * [ (src0) ] --\
 *               +--> [ (sink) ]
 * [ (src1) ] --/
 * </pre>
 */
@Test
public void testNodeHashIdenticalSources() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src0 = env.addSource(new NoOpSourceFunction());
	DataStream<String> src1 = env.addSource(new NoOpSourceFunction());

	src0.union(src1).addSink(new DiscardingSink<>());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	assertTrue(vertices.get(0).isInputVertex());
	assertTrue(vertices.get(1).isInputVertex());

	assertNotNull(vertices.get(0).getID());
	assertNotNull(vertices.get(1).getID());

	assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID());
}
 

private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {
	final JobID jid = new JobID();
	final int parallelism = new Random().nextInt(10) + 1;

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(parallelism);

	// build a simple execution graph with on job vertex, parallelism 2
	final ExecutionGraph graph = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(jg)
		.setRestartStrategy(new InfiniteDelayRestartStrategy())
		.setFailoverStrategyFactory(new CustomStrategy(failoverStrategy))
		.setSlotProvider(slotProvider)
		.build();

	graph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	return graph;
}
 

private static JobGraph createJobGraph(
		ScheduleMode scheduleMode,
		ResultPartitionType resultPartitionType,
		ExecutionMode executionMode) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	JobVertex source = new JobVertex("source");
	source.setInvokableClass(LongValueSource.class);
	source.setParallelism(PARALLELISM);
	source.setSlotSharingGroup(slotSharingGroup);

	JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(ResultVerifyingSink.class);
	sink.setParallelism(PARALLELISM);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
	JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(scheduleMode);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setExecutionMode(executionMode);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

/**
 * Creates an execution graph containing the given vertices and the given restart strategy.
 */
public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		TaskManagerGateway taskManagerGateway,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	int numSlotsNeeded = 0;
	for (JobVertex vertex : vertices) {
		numSlotsNeeded += vertex.getParallelism();
	}

	SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded, taskManagerGateway);

	return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
}
 

@Nonnull
private JobGraph createJobGraphFromJobVerticesWithCheckpointing(SavepointRestoreSettings savepointRestoreSettings, JobVertex... jobVertices) {
	final JobGraph jobGraph = new JobGraph(jobVertices);

	// enable checkpointing which is required to resume from a savepoint
	final CheckpointCoordinatorConfiguration checkpoinCoordinatorConfiguration = new CheckpointCoordinatorConfiguration(
		1000L,
		1000L,
		1000L,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true);
	final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(
		Collections.emptyList(),
		Collections.emptyList(),
		Collections.emptyList(),
		checkpoinCoordinatorConfiguration,
		null);
	jobGraph.setSnapshotSettings(checkpointingSettings);
	jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

	return jobGraph;
}
 

/**
 * Tests that there are no collisions with two identical sources.
 *
 * <pre>
 * [ (src0) ] --\
 *               +--> [ (sink) ]
 * [ (src1) ] --/
 * </pre>
 */
@Test
public void testNodeHashIdenticalSources() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src0 = env.addSource(new NoOpSourceFunction());
	DataStream<String> src1 = env.addSource(new NoOpSourceFunction());

	src0.union(src1).addSink(new DiscardingSink<>());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	assertTrue(vertices.get(0).isInputVertex());
	assertTrue(vertices.get(1).isInputVertex());

	assertNotNull(vertices.get(0).getID());
	assertNotNull(vertices.get(1).getID());

	assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID());
}
 

@Test
public void testInheritOverride() {
	// verify that we can explicitly disable inheritance of the input slot sharing groups

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should not inherit group but be in "default"
	src1.union(src2).filter(dummyFilter).slotSharingGroup("default");
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
}
 

private static JobVertex[] createVerticesForSimpleBipartiteJobGraph(int parallelism, int maxParallelism) {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");

	JobVertex[] jobVertices = new JobVertex[]{v1, v2, v3, v4, v5};

	for (JobVertex jobVertex : jobVertices) {
		jobVertex.setInvokableClass(AbstractInvokable.class);
		jobVertex.setParallelism(parallelism);
		jobVertex.setMaxParallelism(maxParallelism);
	}

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	return jobVertices;
}
 

/**
 * Tests that disabled checkpointing sets the checkpointing interval to Long.MAX_VALUE and the checkpoint mode to
 * {@link CheckpointingMode#AT_LEAST_ONCE}.
 */
@Test
public void testDisabledCheckpointing() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.fromElements(0).print();
	StreamGraph streamGraph = env.getStreamGraph();
	assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled());

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);

	JobCheckpointingSettings snapshottingSettings = jobGraph.getCheckpointingSettings();
	assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointCoordinatorConfiguration().getCheckpointInterval());
	assertFalse(snapshottingSettings.getCheckpointCoordinatorConfiguration().isExactlyOnce());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	StreamConfig streamConfig = new StreamConfig(verticesSorted.get(0).getConfiguration());
	assertEquals(CheckpointingMode.AT_LEAST_ONCE, streamConfig.getCheckpointMode());
}
 

public static Map<JobVertexID, ExecutionJobVertex> includeLegacyJobVertexIDs(
		Map<JobVertexID, ExecutionJobVertex> tasks) {

	Map<JobVertexID, ExecutionJobVertex> expanded = new HashMap<>(2 * tasks.size());
	// first include all new ids
	expanded.putAll(tasks);

	// now expand and add legacy ids
	for (ExecutionJobVertex executionJobVertex : tasks.values()) {
		if (null != executionJobVertex) {
			JobVertex jobVertex = executionJobVertex.getJobVertex();
			if (null != jobVertex) {
				List<JobVertexID> alternativeIds = jobVertex.getIdAlternatives();
				for (JobVertexID jobVertexID : alternativeIds) {
					ExecutionJobVertex old = expanded.put(jobVertexID, executionJobVertex);
					Preconditions.checkState(null == old || old.equals(executionJobVertex),
							"Ambiguous jobvertex id detected during expansion to legacy ids.");
				}
			}
		}
	}

	return expanded;
}
 

private boolean isPartialResourceConfigured(JobGraph jobGraph) {
	boolean hasVerticesWithUnknownResource = false;
	boolean hasVerticesWithConfiguredResource = false;

	for (JobVertex jobVertex : jobGraph.getVertices()) {
		if (jobVertex.getMinResources() == ResourceSpec.UNKNOWN) {
			hasVerticesWithUnknownResource = true;
		} else {
			hasVerticesWithConfiguredResource = true;
		}

		if (hasVerticesWithUnknownResource && hasVerticesWithConfiguredResource) {
			return true;
		}
	}

	return false;
}
 

private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSinkTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	return vertex;
}
 

public static Map<OperatorID, ExecutionJobVertex> includeAlternativeOperatorIDs(
		Map<OperatorID, ExecutionJobVertex> operatorMapping) {

	Map<OperatorID, ExecutionJobVertex> expanded = new HashMap<>(2 * operatorMapping.size());
	// first include all existing ids
	expanded.putAll(operatorMapping);

	// now expand and add user-defined ids
	for (ExecutionJobVertex executionJobVertex : operatorMapping.values()) {
		if (executionJobVertex != null) {
			JobVertex jobVertex = executionJobVertex.getJobVertex();
			if (jobVertex != null) {
				for (OperatorID operatorID : jobVertex.getUserDefinedOperatorIDs()) {
					if (operatorID != null) {
						expanded.put(operatorID, executionJobVertex);
					}
				}
			}
		}
	}

	return expanded;
}
 

public static ExecutionGraph createExecutionGraph(
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(new JobGraph(vertices))
		.setFutureExecutor(executor)
		.setIoExecutor(executor)
		.setSlotProvider(slotProvider)
		.setAllocationTimeout(timeout)
		.setRpcTimeout(timeout)
		.setRestartStrategy(restartStrategy)
		.build();
}
 

/**
 * Creating job graph as below (execution view).
 * It's a representative of streaming job.
 * <pre>
 *     (v11) -+-> (v21)
 *
 *     (v12) -+-> (v22)
 *
 *            ^
 *            |
 *       (pipelined)
 * </pre>
 * 2 regions. Each has 2 pipelined connected vertices.
 */
private JobGraph createStreamingJobGraph() {
	final JobVertex v1 = new JobVertex("vertex1");
	final JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(2);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Testjob", v1, v2);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	return jobGraph;
}
 

/**
 * Verifies that the chain start/end is correctly set.
 */
@Test
public void testChainStartEndSetting() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	// fromElements -> CHAIN(Map -> Print)
	env.fromElements(1, 2, 3)
		.map(new MapFunction<Integer, Integer>() {
			@Override
			public Integer map(Integer value) throws Exception {
				return value;
			}
		})
		.print();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	JobVertex sourceVertex = verticesSorted.get(0);
	JobVertex mapPrintVertex = verticesSorted.get(1);

	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());

	StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
	StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
	Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
	StreamConfig printConfig = chainedConfigs.values().iterator().next();

	assertTrue(sourceConfig.isChainStart());
	assertTrue(sourceConfig.isChainEnd());

	assertTrue(mapConfig.isChainStart());
	assertFalse(mapConfig.isChainEnd());

	assertFalse(printConfig.isChainStart());
	assertTrue(printConfig.isChainEnd());
}
 

public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	return createExecutionGraph(jid, slotProvider, restartStrategy, TestingUtils.defaultExecutor(), vertices);
}
 

@Test
public void testInputConstraintALL() throws Exception {
	List<JobVertex> vertices = createOrderedVertices();
	ExecutionGraph eg = createExecutionGraph(vertices, InputDependencyConstraint.ALL);
	ExecutionVertex ev11 = eg.getJobVertex(vertices.get(0).getID()).getTaskVertices()[0];
	ExecutionVertex ev21 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[0];
	ExecutionVertex ev22 = eg.getJobVertex(vertices.get(1).getID()).getTaskVertices()[1];
	ExecutionVertex ev31 = eg.getJobVertex(vertices.get(2).getID()).getTaskVertices()[0];

	eg.start(mainThreadExecutor);
	eg.scheduleForExecution();


	// Inputs constraint not satisfied on init
	assertFalse(ev31.checkInputDependencyConstraints());

	// Input1 consumable does not satisfy the constraint
	IntermediateResultPartition partition11 = ev11.getProducedPartitions().values().iterator().next();
	ev11.scheduleOrUpdateConsumers(new ResultPartitionID(partition11.getPartitionId(),
		ev11.getCurrentExecutionAttempt().getAttemptId()));
	assertFalse(ev31.checkInputDependencyConstraints());

	// Input2 consumable satisfies the constraint
	ev21.getCurrentExecutionAttempt().markFinished();
	ev22.getCurrentExecutionAttempt().markFinished();
	assertTrue(ev31.checkInputDependencyConstraints());

	// Inputs constraint not satisfied after failover
	ev11.fail(new Exception());


	waitUntilJobRestarted(eg);

	assertFalse(ev31.checkInputDependencyConstraints());
}
 

private JobGraph createTestJobGraph(
		String jobName,
		int senderParallelism,
		int receiverParallelism) {

	// The sender and receiver invokable logic ensure that each subtask gets the expected data
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(RoundRobinSubtaskIndexSender.class);
	sender.getConfiguration().setInteger(RoundRobinSubtaskIndexSender.CONFIG_KEY, receiverParallelism);
	sender.setParallelism(senderParallelism);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SubtaskIndexReceiver.class);
	receiver.getConfiguration().setInteger(SubtaskIndexReceiver.CONFIG_KEY, senderParallelism);
	receiver.setParallelism(receiverParallelism);

	receiver.connectNewDataSetAsInput(
			sender,
			DistributionPattern.ALL_TO_ALL,
			ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(jobName, sender, receiver);

	// We need to allow queued scheduling, because there are not enough slots available
	// to run all tasks at once. We queue tasks and then let them finish/consume the blocking
	// result one after the other.
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

@Test
public void testImmediateReleaseTwoLevel() {
	try {
		JobVertexID vid = new JobVertexID();
		JobVertex vertex = new JobVertex("vertex", vid);
		
		SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
		SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();

		CoLocationGroup coLocationGroup = new CoLocationGroup(vertex);
		CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
		
		Instance instance = SchedulerTestUtils.getRandomInstance(1);
		
		SharedSlot sharedSlot = instance.allocateSharedSlot(assignment);

		SimpleSlot sub = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, constraint);
		
		assertNull(sub.getGroupID());
		assertEquals(constraint.getSharedSlot(), sub.getParent());
		
		sub.releaseSlot();

		assertTrue(sub.isReleased());
		assertTrue(sharedSlot.isReleased());

		assertEquals(1, instance.getNumberOfAvailableSlots());
		assertEquals(0, instance.getNumberOfAllocatedSlots());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private JobGraph producerConsumerJobGraph() {
	final JobVertex producer = new JobVertex("Producer");
	producer.setInvokableClass(NoOpInvokable.class);
	final JobVertex consumer = new JobVertex("Consumer");
	consumer.setInvokableClass(NoOpInvokable.class);

	consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(producer, consumer);
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 

@Test
public void testBipartiteJob() throws Exception {
	final int parallelism = 31;

	final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setNumTaskManagers(1)
		.setNumSlotsPerTaskManager(parallelism)
		.setConfiguration(getDefaultConfiguration())
		.build();

	try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
		miniCluster.start();

		final JobVertex sender = new JobVertex("Sender");
		sender.setInvokableClass(Sender.class);
		sender.setParallelism(parallelism);

		final JobVertex receiver = new JobVertex("Receiver");
		receiver.setInvokableClass(AgnosticReceiver.class);
		receiver.setParallelism(parallelism);

		receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
			ResultPartitionType.PIPELINED);

		final JobGraph jobGraph = new JobGraph("Bipartite Job", sender, receiver);

		miniCluster.executeJobBlocking(jobGraph);
	}
}
 

private static void assertVertexAndConnectedResultsEquals(
	final JobVertex jobVertex,
	final DefaultLogicalVertex logicalVertex) {

	assertVertexInfoEquals(jobVertex, logicalVertex);

	final List<IntermediateDataSet> consumedResults = jobVertex.getInputs().stream()
		.map(JobEdge::getSource)
		.collect(Collectors.toList());
	assertResultsEquals(consumedResults, logicalVertex.getConsumedResults());

	final List<IntermediateDataSet> producedResults = jobVertex.getProducedDataSets();
	assertResultsEquals(producedResults, logicalVertex.getProducedResults());
}