org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex Java Examples

@Test
public void vertexIsResetBeforeRestarted() throws Exception {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();

	final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
	final DefaultScheduler scheduler = createScheduler(jobGraph, schedulingStrategyFactory);
	final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
	final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology();

	startScheduling(scheduler);

	final SchedulingExecutionVertex onlySchedulingVertex = Iterables.getOnlyElement(topology.getVertices());
	schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertex.getId()));

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));

	taskRestartExecutor.triggerScheduledTasks();

	assertThat(schedulingStrategy.getReceivedVerticesToRestart(), hasSize(1));
	assertThat(onlySchedulingVertex.getState(), is(equalTo(ExecutionState.CREATED)));
}
 

@Test
public void failureInfoIsSetAfterTaskFailure() {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final JobID jobId = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();

	final String exceptionMessage = "expected exception";
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, attemptId, ExecutionState.FAILED, new RuntimeException(exceptionMessage)));

	final ErrorInfo failureInfo = scheduler.requestJob().getFailureInfo();
	assertThat(failureInfo, is(notNullValue()));
	assertThat(failureInfo.getExceptionAsString(), containsString(exceptionMessage));
}
 

@Test
public void cancelWhileRestartingShouldWaitForRunningTasks() {
	final JobGraph jobGraph = singleJobVertexJobGraph(2);
	final JobID jobid = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
	final SchedulingTopology topology = scheduler.getSchedulingTopology();

	final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getAllExecutionVertices().iterator();
	final ExecutionAttemptID attemptId1 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();
	final ExecutionAttemptID attemptId2 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();
	final ExecutionVertexID executionVertex2 = scheduler.getExecutionVertexIdOrThrow(attemptId2);

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobid, attemptId1, ExecutionState.FAILED, new RuntimeException("expected")));
	scheduler.cancel();
	final ExecutionState vertex2StateAfterCancel = topology.getVertex(executionVertex2).getState();
	final JobStatus statusAfterCancelWhileRestarting = scheduler.requestJobStatus();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobid, attemptId2, ExecutionState.CANCELED, new RuntimeException("expected")));

	assertThat(vertex2StateAfterCancel, is(equalTo(ExecutionState.CANCELING)));
	assertThat(statusAfterCancelWhileRestarting, is(equalTo(JobStatus.CANCELLING)));
	assertThat(scheduler.requestJobStatus(), is(equalTo(JobStatus.CANCELED)));
}
 

@Test
public void jobStatusIsRestartingIfOneVertexIsWaitingForRestart() {
	final JobGraph jobGraph = singleJobVertexJobGraph(2);
	final JobID jobId = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getAllExecutionVertices().iterator();
	final ExecutionAttemptID attemptId1 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();
	final ExecutionAttemptID attemptId2 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, attemptId1, ExecutionState.FAILED, new RuntimeException("expected")));
	final JobStatus jobStatusAfterFirstFailure = scheduler.requestJobStatus();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, attemptId2, ExecutionState.FAILED, new RuntimeException("expected")));

	taskRestartExecutor.triggerNonPeriodicScheduledTask();
	final JobStatus jobStatusWithPendingRestarts = scheduler.requestJobStatus();
	taskRestartExecutor.triggerNonPeriodicScheduledTask();
	final JobStatus jobStatusAfterRestarts = scheduler.requestJobStatus();

	assertThat(jobStatusAfterFirstFailure, equalTo(JobStatus.RESTARTING));
	assertThat(jobStatusWithPendingRestarts, equalTo(JobStatus.RESTARTING));
	assertThat(jobStatusAfterRestarts, equalTo(JobStatus.RUNNING));
}
 

@Test
public void abortPendingCheckpointsWhenRestartingTasks() throws Exception {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	enableCheckpointing(jobGraph);

	final CountDownLatch checkpointTriggeredLatch = getCheckpointTriggeredLatch();

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.RUNNING));

	final CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(scheduler);

	checkpointCoordinator.triggerCheckpoint(false);
	checkpointTriggeredLatch.await();
	assertThat(checkpointCoordinator.getNumberOfPendingCheckpoints(), is(equalTo(1)));

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));
	taskRestartExecutor.triggerScheduledTasks();
	assertThat(checkpointCoordinator.getNumberOfPendingCheckpoints(), is(equalTo(0)));
}
 

@Test
public void handleGlobalFailure() {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph);

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	scheduler.handleGlobalFailure(new Exception("forced failure"));

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.CANCELED));

	taskRestartExecutor.triggerScheduledTasks();

	final List<ExecutionVertexID> deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices();
	final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0);
	assertThat(deployedExecutionVertices, contains(executionVertexId, executionVertexId));
}
 

@Test
public void skipDeploymentIfVertexVersionOutdated() {
	testExecutionSlotAllocator.disableAutoCompletePendingRequests();

	final JobGraph jobGraph = nonParallelSourceSinkJobGraph();
	final List<JobVertex> sortedJobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	final ExecutionVertexID sourceExecutionVertexId = new ExecutionVertexID(sortedJobVertices.get(0).getID(), 0);
	final ExecutionVertexID sinkExecutionVertexId = new ExecutionVertexID(sortedJobVertices.get(1).getID(), 0);

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
	testExecutionSlotAllocator.completePendingRequest(sourceExecutionVertexId);

	final ArchivedExecutionVertex sourceExecutionVertex = scheduler.requestJob().getAllExecutionVertices().iterator().next();
	final ExecutionAttemptID attemptId = sourceExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));
	testRestartBackoffTimeStrategy.setCanRestart(false);

	testExecutionSlotAllocator.enableAutoCompletePendingRequests();
	taskRestartExecutor.triggerScheduledTasks();

	assertThat(testExecutionVertexOperations.getDeployedVertices(), containsInAnyOrder(sourceExecutionVertexId, sinkExecutionVertexId));
	assertThat(scheduler.requestJob().getState(), is(equalTo(JobStatus.RUNNING)));
}
 

@Test
public void failJobIfCannotRestart() throws Exception {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	testRestartBackoffTimeStrategy.setCanRestart(false);

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));

	taskRestartExecutor.triggerScheduledTasks();

	waitForTermination(scheduler);
	final JobStatus jobStatus = scheduler.requestJobStatus();
	assertThat(jobStatus, is(equalTo(JobStatus.FAILED)));
}
 

@Test
public void restartFailedTask() {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph);

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex archivedExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = archivedExecutionVertex.getCurrentExecutionAttempt().getAttemptId();

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));

	taskRestartExecutor.triggerScheduledTasks();

	final List<ExecutionVertexID> deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices();
	final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0);
	assertThat(deployedExecutionVertices, contains(executionVertexId, executionVertexId));
}
 

public ArchivedExecutionVertex build() {
	Preconditions.checkNotNull(currentExecution);
	return new ArchivedExecutionVertex(
		subtaskIndex,
		taskNameWithSubtask != null ? taskNameWithSubtask : "task_" + RANDOM.nextInt() + "_" + subtaskIndex,
		currentExecution,
		priorExecutions != null ? priorExecutions : new EvictingBoundedList<ArchivedExecution>(0)
	);
}
 

private static ArchivedExecutionJobVertex createArchivedExecutionJobVertex(JobVertexID jobVertexID) {
	final StringifiedAccumulatorResult[] emptyAccumulators = new StringifiedAccumulatorResult[0];
	final long[] timestamps = new long[ExecutionState.values().length];
	final ExecutionState expectedState = ExecutionState.RUNNING;

	final LocalTaskManagerLocation assignedResourceLocation = new LocalTaskManagerLocation();
	final AllocationID allocationID = new AllocationID();

	final int subtaskIndex = 1;
	final int attempt = 2;
	return new ArchivedExecutionJobVertex(
		new ArchivedExecutionVertex[]{
			new ArchivedExecutionVertex(
				subtaskIndex,
				"test task",
				new ArchivedExecution(
					new StringifiedAccumulatorResult[0],
					null,
					new ExecutionAttemptID(),
					attempt,
					expectedState,
					"error",
					assignedResourceLocation,
					allocationID,
					subtaskIndex,
					timestamps),
				new EvictingBoundedList<>(0)
			)
		},
		jobVertexID,
		jobVertexID.toString(),
		1,
		1,
		ResourceProfile.UNKNOWN,
		emptyAccumulators);
}
 

@Test
public void vertexIsNotAffectedByOutdatedDeployment() {
	final JobGraph jobGraph = singleJobVertexJobGraph(2);

	testExecutionSlotAllocator.disableAutoCompletePendingRequests();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getAllExecutionVertices().iterator();
	final ArchivedExecutionVertex v1 = vertexIterator.next();
	final ArchivedExecutionVertex v2 = vertexIterator.next();

	final SchedulingExecutionVertex sv1 = scheduler.getSchedulingTopology().getVertices().iterator().next();

	// fail v1 and let it recover to SCHEDULED
	// the initial deployment of v1 will be outdated
	scheduler.updateTaskExecutionState(new TaskExecutionState(
		jobGraph.getJobID(),
		v1.getCurrentExecutionAttempt().getAttemptId(),
		ExecutionState.FAILED));
	taskRestartExecutor.triggerScheduledTasks();

	// fail v2 to get all pending slot requests in the initial deployments to be done
	// this triggers the outdated deployment of v1
	scheduler.updateTaskExecutionState(new TaskExecutionState(
		jobGraph.getJobID(),
		v2.getCurrentExecutionAttempt().getAttemptId(),
		ExecutionState.FAILED));

	// v1 should not be affected
	assertThat(sv1.getState(), is(equalTo(ExecutionState.SCHEDULED)));
}
 

@Test
public void restoreStateWhenRestartingTasks() throws Exception {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	enableCheckpointing(jobGraph);

	final CountDownLatch checkpointTriggeredLatch = getCheckpointTriggeredLatch();

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.RUNNING));

	final CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(scheduler);

	// register a stateful master hook to help verify state restore
	final TestMasterHook masterHook = TestMasterHook.fromId("testHook");
	checkpointCoordinator.addMasterHook(masterHook);

	// complete one checkpoint for state restore
	checkpointCoordinator.triggerCheckpoint(false);
	checkpointTriggeredLatch.await();
	final long checkpointId = checkpointCoordinator.getPendingCheckpoints().keySet().iterator().next();
	acknowledgePendingCheckpoint(scheduler, checkpointId);

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));
	taskRestartExecutor.triggerScheduledTasks();
	assertThat(masterHook.getRestoreCount(), is(equalTo(1)));
}
 

@Test
public void failGlobalWhenRestoringStateFails() throws Exception {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph);
	enableCheckpointing(jobGraph);

	final CountDownLatch checkpointTriggeredLatch = getCheckpointTriggeredLatch();

	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.RUNNING));

	final CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(scheduler);

	// register a master hook to fail state restore
	final TestMasterHook masterHook = TestMasterHook.fromId("testHook");
	masterHook.enableFailOnRestore();
	checkpointCoordinator.addMasterHook(masterHook);

	// complete one checkpoint for state restore
	checkpointCoordinator.triggerCheckpoint(false);
	checkpointTriggeredLatch.await();
	final long checkpointId = checkpointCoordinator.getPendingCheckpoints().keySet().iterator().next();
	acknowledgePendingCheckpoint(scheduler, checkpointId);

	scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED));
	taskRestartExecutor.triggerScheduledTasks();
	final List<ExecutionVertexID> deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices();

	// the first task failover should be skipped on state restore failure
	final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0);
	assertThat(deployedExecutionVertices, contains(executionVertexId));

	// a global failure should be triggered on state restore failure
	masterHook.disableFailOnRestore();
	taskRestartExecutor.triggerScheduledTasks();
	assertThat(deployedExecutionVertices, contains(executionVertexId, executionVertexId));
}
 

public ArchivedExecutionVertex build() {
	Preconditions.checkNotNull(currentExecution);
	return new ArchivedExecutionVertex(
		subtaskIndex,
		taskNameWithSubtask != null ? taskNameWithSubtask : "task_" + RANDOM.nextInt() + "_" + subtaskIndex,
		currentExecution,
		priorExecutions != null ? priorExecutions : new EvictingBoundedList<ArchivedExecution>(0)
	);
}
 

private static void generateArchivedJob() throws Exception {
	// Attempt
	StringifiedAccumulatorResult acc1 = new StringifiedAccumulatorResult("name1", "type1", "value1");
	StringifiedAccumulatorResult acc2 = new StringifiedAccumulatorResult("name2", "type2", "value2");
	TaskManagerLocation location = new TaskManagerLocation(new ResourceID("hello"), InetAddress.getLocalHost(), 1234);
	AllocationID allocationID = new AllocationID(42L, 43L);
	originalAttempt = new ArchivedExecutionBuilder()
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9})
		.setParallelSubtaskIndex(1)
		.setAttemptNumber(0)
		.setAssignedResourceLocation(location)
		.setAssignedAllocationID(allocationID)
		.setUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.setState(ExecutionState.FINISHED)
		.setFailureCause("attemptException")
		.build();
	// Subtask
	originalSubtask = new ArchivedExecutionVertexBuilder()
		.setSubtaskIndex(originalAttempt.getParallelSubtaskIndex())
		.setTaskNameWithSubtask("hello(1/1)")
		.setCurrentExecution(originalAttempt)
		.build();
	// Task
	originalTask = new ArchivedExecutionJobVertexBuilder()
		.setTaskVertices(new ArchivedExecutionVertex[]{originalSubtask})
		.build();
	// Job
	Map<JobVertexID, ArchivedExecutionJobVertex> tasks = new HashMap<>();
	tasks.put(originalTask.getJobVertexId(), originalTask);
	originalJob = new ArchivedExecutionGraphBuilder()
		.setJobID(new JobID())
		.setTasks(tasks)
		.setFailureCause(new ErrorInfo(new Exception("jobException"), originalAttempt.getStateTimestamp(ExecutionState.FAILED)))
		.setState(JobStatus.FINISHED)
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
		.setArchivedUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.build();
}
 

public ArchivedExecutionJobVertexBuilder setTaskVertices(ArchivedExecutionVertex[] taskVertices) {
	this.taskVertices = taskVertices;
	return this;
}
 

public ArchivedExecutionJobVertexBuilder setTaskVertices(ArchivedExecutionVertex[] taskVertices) {
	this.taskVertices = taskVertices;
	return this;
}