org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
	onlyExecutionVertex.deployToSlot(testingLogicalSlot);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertTrue(taskManagerLocationOptional.isPresent());

	final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
	assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}
Example #2
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that it will throw exception when getting the task manager location of a non existing execution.
 */
@Test
public void testGetNonExistingExecutionVertexWillThrowException() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID invalidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	try {
		inputsLocationsRetriever.getTaskManagerLocation(invalidExecutionVertexId);
		fail("Should throw exception if execution vertex doesn't exist!");
	} catch (IllegalStateException expected) {
		// expect this exception
	}
}
Example #3
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
	onlyExecutionVertex.deployToSlot(testingLogicalSlot);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertTrue(taskManagerLocationOptional.isPresent());

	final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
	assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}
Example #4
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that it will throw exception when getting the task manager location of a non existing execution.
 */
@Test
public void testGetNonExistingExecutionVertexWillThrowException() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID invalidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	try {
		inputsLocationsRetriever.getTaskManagerLocation(invalidExecutionVertexId);
		fail("Should throw exception if execution vertex doesn't exist!");
	} catch (IllegalStateException expected) {
		// expect this exception
	}
}
Example #5
Source File: TaskExecutorITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #6
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	final Time timeout = Time.days(1L);

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);

	final ExecutionGraph executionGraph = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobVertex)
		.setRpcTimeout(timeout)
		.setAllocationTimeout(timeout)
		.allowQueuedScheduling()
		.build();

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);

	executionGraph.enableCheckpointing(
			chkConfig,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	return executionGraph;
}
Example #7
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #8
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that can get the producers of consumed result partitions.
 */
@Test
public void testGetConsumedResultPartitionsProducers() throws Exception {
	final JobVertex producer1 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex producer2 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex consumer = ExecutionGraphTestUtils.createNoOpVertex(1);
	consumer.connectNewDataSetAsInput(producer1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	consumer.connectNewDataSetAsInput(producer2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), producer1, producer2, consumer);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID evIdOfProducer1 = new ExecutionVertexID(producer1.getID(), 0);
	ExecutionVertexID evIdOfProducer2 = new ExecutionVertexID(producer2.getID(), 0);
	ExecutionVertexID evIdOfConsumer = new ExecutionVertexID(consumer.getID(), 0);

	Collection<Collection<ExecutionVertexID>> producersOfProducer1 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer1);
	Collection<Collection<ExecutionVertexID>> producersOfProducer2 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer2);
	Collection<Collection<ExecutionVertexID>> producersOfConsumer =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfConsumer);

	assertThat(producersOfProducer1, is(empty()));
	assertThat(producersOfProducer2, is(empty()));
	assertThat(producersOfConsumer, hasSize(2));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer1)));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer2)));
}
Example #9
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that it will get empty task manager location if vertex is not scheduled.
 */
@Test
public void testGetEmptyTaskManagerLocationIfVertexNotScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocation =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertFalse(taskManagerLocation.isPresent());
}
Example #10
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
Example #11
Source File: DefaultSchedulingPipelinedRegionTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests if the consumed inputs of the pipelined regions are computed
 * correctly using the Job graph below.
 * <pre>
 *          c
 *        /  X
 * a -+- b   e
 *       \  /
 *        d
 * </pre>
 * Pipelined regions: {a}, {b, c, d, e}
 */
@Test
public void returnsIncidentBlockingPartitions() throws Exception {
	final JobVertex a = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex b = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex c = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex d = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex e = ExecutionGraphTestUtils.createNoOpVertex(1);

	b.connectNewDataSetAsInput(a, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
	c.connectNewDataSetAsInput(b, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	d.connectNewDataSetAsInput(b, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	e.connectNewDataSetAsInput(c, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
	e.connectNewDataSetAsInput(d, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(a, b, c, d, e);
	final DefaultExecutionTopology topology = new DefaultExecutionTopology(simpleTestGraph);

	final DefaultSchedulingPipelinedRegion firstPipelinedRegion = topology.getPipelinedRegionOfVertex(new ExecutionVertexID(a.getID(), 0));
	final DefaultSchedulingPipelinedRegion secondPipelinedRegion = topology.getPipelinedRegionOfVertex(new ExecutionVertexID(e.getID(), 0));

	final DefaultExecutionVertex vertexB0 = topology.getVertex(new ExecutionVertexID(b.getID(), 0));
	final IntermediateResultPartitionID b0ConsumedResultPartition = Iterables.getOnlyElement(vertexB0.getConsumedResults()).getId();

	final Set<IntermediateResultPartitionID> secondPipelinedRegionConsumedResults = IterableUtils.toStream(secondPipelinedRegion.getConsumedResults())
		.map(DefaultResultPartition::getId)
		.collect(Collectors.toSet());

	assertThat(firstPipelinedRegion.getConsumedResults().iterator().hasNext(), is(false));
	assertThat(secondPipelinedRegionConsumedResults, contains(b0ConsumedResultPartition));
}
Example #12
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that can get the producers of consumed result partitions.
 */
@Test
public void testGetConsumedResultPartitionsProducers() throws Exception {
	final JobVertex producer1 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex producer2 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex consumer = ExecutionGraphTestUtils.createNoOpVertex(1);
	consumer.connectNewDataSetAsInput(producer1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	consumer.connectNewDataSetAsInput(producer2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(producer1, producer2, consumer);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID evIdOfProducer1 = new ExecutionVertexID(producer1.getID(), 0);
	ExecutionVertexID evIdOfProducer2 = new ExecutionVertexID(producer2.getID(), 0);
	ExecutionVertexID evIdOfConsumer = new ExecutionVertexID(consumer.getID(), 0);

	Collection<Collection<ExecutionVertexID>> producersOfProducer1 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer1);
	Collection<Collection<ExecutionVertexID>> producersOfProducer2 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer2);
	Collection<Collection<ExecutionVertexID>> producersOfConsumer =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfConsumer);

	assertThat(producersOfProducer1, is(empty()));
	assertThat(producersOfProducer2, is(empty()));
	assertThat(producersOfConsumer, hasSize(2));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer1)));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer2)));
}
Example #13
Source File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that it will get empty task manager location if vertex is not scheduled.
 */
@Test
public void testGetEmptyTaskManagerLocationIfVertexNotScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocation =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertFalse(taskManagerLocation.isPresent());
}
Example #14
Source File: JobMasterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRequestNextInputSplit() throws Exception {
	final List<TestingInputSplit> expectedInputSplits = Arrays.asList(
		new TestingInputSplit(1),
		new TestingInputSplit(42),
		new TestingInputSplit(1337));

	// build one node JobGraph
	InputSplitSource<TestingInputSplit> inputSplitSource = new TestingInputSplitSource(expectedInputSplits);

	JobVertex source = new JobVertex("vertex1");
	source.setParallelism(1);
	source.setInputSplitSource(inputSplitSource);
	source.setInvokableClass(AbstractInvokable.class);

	final JobGraph testJobGraph = new JobGraph(source);
	testJobGraph.setAllowQueuedScheduling(true);

	configuration.setLong(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	configuration.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	final JobManagerSharedServices jobManagerSharedServices =
		new TestingJobManagerSharedServicesBuilder()
			.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
			.build();

	final JobMaster jobMaster = createJobMaster(
		configuration,
		testJobGraph,
		haServices,
		jobManagerSharedServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

		ExecutionGraph eg = jobMaster.getExecutionGraph();

		ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

		final SupplierWithException<SerializedInputSplit, Exception> inputSplitSupplier = () -> jobMasterGateway.requestNextInputSplit(
			source.getID(),
			ev.getCurrentExecutionAttempt().getAttemptId()).get();

		List<InputSplit> actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		final Matcher<Iterable<? extends InputSplit>> expectedInputSplitsMatcher = containsInAnyOrder(expectedInputSplits.toArray(EMPTY_TESTING_INPUT_SPLITS));
		assertThat(actualInputSplits, expectedInputSplitsMatcher);

		final long maxWaitMillis = 2000L;
		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		CompletableFuture.runAsync(() -> eg.failGlobal(new Exception("Testing exception")), eg.getJobMasterMainThreadExecutor()).get();

		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		assertThat(actualInputSplits, expectedInputSplitsMatcher);
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}