Java Code Examples for org.apache.flink.runtime.jobgraph.JobVertex#setStrictlyCoLocatedWith()

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testBlockingAllToAllTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(13);

	target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(region1 == region2);
}
 

/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 

/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testBlockingAllToAllTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(13);

	target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(region1 == region2);
}
 

/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 

@Test
public void coLocationConstraintIsResetOnTaskRecovery() {
	final JobGraph jobGraph = nonParallelSourceSinkJobGraph();
	final JobVertex source = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
	final JobVertex sink = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);

	final SlotSharingGroup ssg = new SlotSharingGroup();
	source.setSlotSharingGroup(ssg);
	sink.setSlotSharingGroup(ssg);
	sink.setStrictlyCoLocatedWith(source);

	final JobID jobId = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ExecutionVertex sourceVertex = scheduler.getExecutionVertex(new ExecutionVertexID(source.getID(), 0));
	final ExecutionAttemptID sourceAttemptId = sourceVertex.getCurrentExecutionAttempt().getAttemptId();
	final ExecutionVertex sinkVertex = scheduler.getExecutionVertex(new ExecutionVertexID(sink.getID(), 0));
	final ExecutionAttemptID sinkAttemptId = sinkVertex.getCurrentExecutionAttempt().getAttemptId();

	// init the location constraint manually because the testExecutionSlotAllocator does not do it
	sourceVertex.getLocationConstraint().setSlotRequestId(new SlotRequestId());
	assertThat(sourceVertex.getLocationConstraint().getSlotRequestId(), is(notNullValue()));

	final String exceptionMessage = "expected exception";
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, sourceAttemptId, ExecutionState.FAILED, new RuntimeException(exceptionMessage)));
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, sinkAttemptId, ExecutionState.CANCELED));
	taskRestartExecutor.triggerScheduledTasks();

	assertThat(sourceVertex.getLocationConstraint().getSlotRequestId(), is(nullValue()));
}
 

@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		testingSlotProvider,
		new TestRestartStrategy(
			1,
			false),
		groupVertex,
		groupVertex2);

	// enable the queued scheduling for the slot pool
	eg.setQueuedSchedulingAllowed(true);
	eg.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}
 

@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(groupVertex, groupVertex2)
		.setSlotProvider(testingSlotProvider)
		.setRestartStrategy(
			new TestRestartStrategy(
				1,
				false))
		.allowQueuedScheduling()
		.build();

	// enable the queued scheduling for the slot pool
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}
 

@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(new JobGraph(groupVertex, groupVertex2))
		.setSlotProvider(testingSlotProvider)
		.setRestartStrategy(new TestRestartStrategy(1, false))
		.build();

	// enable the queued scheduling for the slot pool
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}