Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionGraph#getAllExecutionVertices()

public ExecutionGraphToSchedulingTopologyAdapter(ExecutionGraph graph) {
	checkNotNull(graph, "execution graph can not be null");

	this.executionVerticesById = new HashMap<>();
	this.executionVerticesList = new ArrayList<>(graph.getTotalNumberOfVertices());
	Map<IntermediateResultPartitionID, DefaultSchedulingResultPartition> tmpResultPartitionsById = new HashMap<>();
	Map<ExecutionVertex, DefaultSchedulingExecutionVertex> executionVertexMap = new HashMap<>();

	for (ExecutionVertex vertex : graph.getAllExecutionVertices()) {
		List<DefaultSchedulingResultPartition> producedPartitions = generateProducedSchedulingResultPartition(vertex.getProducedPartitions());

		producedPartitions.forEach(partition -> tmpResultPartitionsById.put(partition.getId(), partition));

		DefaultSchedulingExecutionVertex schedulingVertex = generateSchedulingExecutionVertex(vertex, producedPartitions);
		this.executionVerticesById.put(schedulingVertex.getId(), schedulingVertex);
		this.executionVerticesList.add(schedulingVertex);
		executionVertexMap.put(vertex, schedulingVertex);
	}
	this.resultPartitionsById = tmpResultPartitionsById;

	connectVerticesToConsumedPartitions(executionVertexMap, tmpResultPartitionsById);
}
 

public DefaultFailoverTopology(ExecutionGraph executionGraph) {
	checkNotNull(executionGraph);

	this.containsCoLocationConstraints = executionGraph.getAllVertices().values().stream()
		.map(ExecutionJobVertex::getCoLocationGroup)
		.anyMatch(Objects::nonNull);

	// generate vertices
	this.failoverVertices = new ArrayList<>();
	final Map<ExecutionVertex, DefaultFailoverVertex> failoverVertexMap = new IdentityHashMap<>();
	for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
		final DefaultFailoverVertex failoverVertex = new DefaultFailoverVertex(
			new ExecutionVertexID(vertex.getJobvertexId(), vertex.getParallelSubtaskIndex()),
			vertex.getTaskNameWithSubtaskIndex());
		this.failoverVertices.add(failoverVertex);
		failoverVertexMap.put(vertex, failoverVertex);
	}

	// generate edges
	connectVerticesWithEdges(failoverVertexMap);
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
 

public DefaultExecutionTopology(ExecutionGraph graph) {
	checkNotNull(graph, "execution graph can not be null");

	this.containsCoLocationConstraints = graph.getAllVertices().values().stream()
		.map(ExecutionJobVertex::getCoLocationGroup)
		.anyMatch(Objects::nonNull);

	this.executionVerticesById = new HashMap<>();
	this.executionVerticesList = new ArrayList<>(graph.getTotalNumberOfVertices());
	Map<IntermediateResultPartitionID, DefaultResultPartition> tmpResultPartitionsById = new HashMap<>();
	Map<ExecutionVertex, DefaultExecutionVertex> executionVertexMap = new HashMap<>();

	for (ExecutionVertex vertex : graph.getAllExecutionVertices()) {
		List<DefaultResultPartition> producedPartitions = generateProducedSchedulingResultPartition(vertex.getProducedPartitions());

		producedPartitions.forEach(partition -> tmpResultPartitionsById.put(partition.getId(), partition));

		DefaultExecutionVertex schedulingVertex = generateSchedulingExecutionVertex(vertex, producedPartitions);
		this.executionVerticesById.put(schedulingVertex.getId(), schedulingVertex);
		this.executionVerticesList.add(schedulingVertex);
		executionVertexMap.put(vertex, schedulingVertex);
	}
	this.resultPartitionsById = tmpResultPartitionsById;

	connectVerticesToConsumedPartitions(executionVertexMap, tmpResultPartitionsById);

	this.pipelinedRegionsByVertex = new HashMap<>();
	this.pipelinedRegions = new ArrayList<>();
	initializePipelinedRegions();
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}