Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionVertex#getCurrentExecutionAttempt()

/**
 * Check if all tasks that need to acknowledge the checkpoint are running.
 * If not, abort the checkpoint
 *
 * @return the execution vertices which should give an ack response
 * @throws CheckpointException the exception fails checking
 */
private Map<ExecutionAttemptID, ExecutionVertex> getAckTasks() throws CheckpointException {
	Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(tasksToWaitFor.length);

	for (ExecutionVertex ev : tasksToWaitFor) {
		Execution ee = ev.getCurrentExecutionAttempt();
		if (ee != null) {
			ackTasks.put(ee.getAttemptId(), ev);
		} else {
			LOG.info(
				"Checkpoint acknowledging task {} of job {} is not being executed at the moment. Aborting checkpoint.",
				ev.getTaskNameWithSubtaskIndex(),
				job);
			throw new CheckpointException(
				CheckpointFailureReason.NOT_ALL_REQUIRED_TASKS_RUNNING);
		}
	}
	return ackTasks;
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
 

private void sendAcknowledgeMessages(long checkpointId, long timestamp) {
	// commit tasks
	for (ExecutionVertex ev : tasksToCommitTo) {
		Execution ee = ev.getCurrentExecutionAttempt();
		if (ee != null) {
			ee.notifyCheckpointComplete(checkpointId, timestamp);
		}
	}

	// commit coordinators
	for (OperatorCoordinatorCheckpointContext coordinatorContext : coordinatorsToCheckpoint) {
		coordinatorContext.checkpointComplete(checkpointId);
	}
}
 

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
 

/**
 * Try to complete the given pending checkpoint.
 *
 * <p>Important: This method should only be called in the checkpoint lock scope.
 *
 * @param pendingCheckpoint to complete
 * @throws CheckpointException if the completion failed
 */
private void completePendingCheckpoint(PendingCheckpoint pendingCheckpoint) throws CheckpointException {
	final long checkpointId = pendingCheckpoint.getCheckpointId();
	final CompletedCheckpoint completedCheckpoint;

	// As a first step to complete the checkpoint, we register its state with the registry
	Map<OperatorID, OperatorState> operatorStates = pendingCheckpoint.getOperatorStates();
	sharedStateRegistry.registerAll(operatorStates.values());

	try {
		try {
			completedCheckpoint = pendingCheckpoint.finalizeCheckpoint();
		}
		catch (Exception e1) {
			// abort the current pending checkpoint if we fails to finalize the pending checkpoint.
			if (!pendingCheckpoint.isDiscarded()) {
				pendingCheckpoint.abortError(e1);
			}

			throw new CheckpointException("Could not finalize the pending checkpoint " + checkpointId + '.', e1);
		}

		// the pending checkpoint must be discarded after the finalization
		Preconditions.checkState(pendingCheckpoint.isDiscarded() && completedCheckpoint != null);

		try {
			completedCheckpointStore.addCheckpoint(completedCheckpoint);
		} catch (Exception exception) {
			// we failed to store the completed checkpoint. Let's clean up
			executor.execute(new Runnable() {
				@Override
				public void run() {
					try {
						completedCheckpoint.discardOnFailedStoring();
					} catch (Throwable t) {
						LOG.warn("Could not properly discard completed checkpoint {}.", completedCheckpoint.getCheckpointID(), t);
					}
				}
			});

			throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.', exception);
		}
	} finally {
		pendingCheckpoints.remove(checkpointId);

		triggerQueuedRequests();
	}

	rememberRecentCheckpointId(checkpointId);

	// drop those pending checkpoints that are at prior to the completed one
	dropSubsumedCheckpoints(checkpointId);

	// record the time when this was completed, to calculate
	// the 'min delay between checkpoints'
	lastCheckpointCompletionNanos = System.nanoTime();

	LOG.info("Completed checkpoint {} for job {} ({} bytes in {} ms).", checkpointId, job,
		completedCheckpoint.getStateSize(), completedCheckpoint.getDuration());

	if (LOG.isDebugEnabled()) {
		StringBuilder builder = new StringBuilder();
		builder.append("Checkpoint state: ");
		for (OperatorState state : completedCheckpoint.getOperatorStates().values()) {
			builder.append(state);
			builder.append(", ");
		}
		// Remove last two chars ", "
		builder.setLength(builder.length() - 2);

		LOG.debug(builder.toString());
	}

	// send the "notify complete" call to all vertices
	final long timestamp = completedCheckpoint.getTimestamp();

	for (ExecutionVertex ev : tasksToCommitTo) {
		Execution ee = ev.getCurrentExecutionAttempt();
		if (ee != null) {
			ee.notifyCheckpointComplete(checkpointId, timestamp);
		}
	}
}
 

private void testMaxConcurrentAttempts(int maxConcurrentAttempts) {
	try {
		final JobID jid = new JobID();

		// create some mock execution vertices and trigger some checkpoint
		final ExecutionAttemptID triggerAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID ackAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID commitAttemptID = new ExecutionAttemptID();

		ExecutionVertex triggerVertex = mockExecutionVertex(triggerAttemptID);
		ExecutionVertex ackVertex = mockExecutionVertex(ackAttemptID);
		ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);

		final AtomicInteger numCalls = new AtomicInteger();

		final Execution execution = triggerVertex.getCurrentExecutionAttempt();

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).notifyCheckpointComplete(anyLong(), anyLong());

		CheckpointCoordinator coord = new CheckpointCoordinator(
			jid,
			10,        // periodic interval is 10 ms
			200000,    // timeout is very long (200 s)
			0L,        // no extra delay
			maxConcurrentAttempts,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			new ExecutionVertex[] { triggerVertex },
			new ExecutionVertex[] { ackVertex },
			new ExecutionVertex[] { commitVertex },
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(2),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY);

		coord.startCheckpointScheduler();

		// after a while, there should be exactly as many checkpoints
		// as concurrently permitted
		long now = System.currentTimeMillis();
		long timeout = now + 60000;
		long minDuration = now + 100;
		do {
			Thread.sleep(20);
		}
		while ((now = System.currentTimeMillis()) < minDuration ||
				(numCalls.get() < maxConcurrentAttempts && now < timeout));

		assertEquals(maxConcurrentAttempts, numCalls.get());

		verify(triggerVertex.getCurrentExecutionAttempt(), times(maxConcurrentAttempts))
				.triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		// now, once we acknowledge one checkpoint, it should trigger the next one
		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID, 1L));

		// this should have immediately triggered a new checkpoint
		now = System.currentTimeMillis();
		timeout = now + 60000;
		do {
			Thread.sleep(20);
		}
		while (numCalls.get() < maxConcurrentAttempts + 1 && now < timeout);

		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		// no further checkpoints should happen
		Thread.sleep(200);
		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		coord.shutdown(JobStatus.FINISHED);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Try to complete the given pending checkpoint.
 *
 * <p>Important: This method should only be called in the checkpoint lock scope.
 *
 * @param pendingCheckpoint to complete
 * @throws CheckpointException if the completion failed
 */
private void completePendingCheckpoint(PendingCheckpoint pendingCheckpoint) throws CheckpointException {
	final long checkpointId = pendingCheckpoint.getCheckpointId();
	final CompletedCheckpoint completedCheckpoint;

	// As a first step to complete the checkpoint, we register its state with the registry
	Map<OperatorID, OperatorState> operatorStates = pendingCheckpoint.getOperatorStates();
	sharedStateRegistry.registerAll(operatorStates.values());

	try {
		try {
			completedCheckpoint = pendingCheckpoint.finalizeCheckpoint();
			failureManager.handleCheckpointSuccess(pendingCheckpoint.getCheckpointId());
		}
		catch (Exception e1) {
			// abort the current pending checkpoint if we fails to finalize the pending checkpoint.
			if (!pendingCheckpoint.isDiscarded()) {
				failPendingCheckpoint(pendingCheckpoint, CheckpointFailureReason.FINALIZE_CHECKPOINT_FAILURE, e1);
			}

			throw new CheckpointException("Could not finalize the pending checkpoint " + checkpointId + '.',
				CheckpointFailureReason.FINALIZE_CHECKPOINT_FAILURE, e1);
		}

		// the pending checkpoint must be discarded after the finalization
		Preconditions.checkState(pendingCheckpoint.isDiscarded() && completedCheckpoint != null);

		try {
			completedCheckpointStore.addCheckpoint(completedCheckpoint);
		} catch (Exception exception) {
			// we failed to store the completed checkpoint. Let's clean up
			executor.execute(new Runnable() {
				@Override
				public void run() {
					try {
						completedCheckpoint.discardOnFailedStoring();
					} catch (Throwable t) {
						LOG.warn("Could not properly discard completed checkpoint {}.", completedCheckpoint.getCheckpointID(), t);
					}
				}
			});

			throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.',
				CheckpointFailureReason.FINALIZE_CHECKPOINT_FAILURE, exception);
		}
	} finally {
		pendingCheckpoints.remove(checkpointId);

		triggerQueuedRequests();
	}

	rememberRecentCheckpointId(checkpointId);

	// drop those pending checkpoints that are at prior to the completed one
	dropSubsumedCheckpoints(checkpointId);

	// record the time when this was completed, to calculate
	// the 'min delay between checkpoints'
	lastCheckpointCompletionNanos = System.nanoTime();

	LOG.info("Completed checkpoint {} for job {} ({} bytes in {} ms).", checkpointId, job,
		completedCheckpoint.getStateSize(), completedCheckpoint.getDuration());

	if (LOG.isDebugEnabled()) {
		StringBuilder builder = new StringBuilder();
		builder.append("Checkpoint state: ");
		for (OperatorState state : completedCheckpoint.getOperatorStates().values()) {
			builder.append(state);
			builder.append(", ");
		}
		// Remove last two chars ", "
		builder.setLength(builder.length() - 2);

		LOG.debug(builder.toString());
	}

	// send the "notify complete" call to all vertices
	final long timestamp = completedCheckpoint.getTimestamp();

	for (ExecutionVertex ev : tasksToCommitTo) {
		Execution ee = ev.getCurrentExecutionAttempt();
		if (ee != null) {
			ee.notifyCheckpointComplete(checkpointId, timestamp);
		}
	}
}
 

private void testMaxConcurrentAttempts(int maxConcurrentAttempts) {
	try {
		final JobID jid = new JobID();

		// create some mock execution vertices and trigger some checkpoint
		final ExecutionAttemptID triggerAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID ackAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID commitAttemptID = new ExecutionAttemptID();

		ExecutionVertex triggerVertex = mockExecutionVertex(triggerAttemptID);
		ExecutionVertex ackVertex = mockExecutionVertex(ackAttemptID);
		ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);

		final AtomicInteger numCalls = new AtomicInteger();

		final Execution execution = triggerVertex.getCurrentExecutionAttempt();

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).notifyCheckpointComplete(anyLong(), anyLong());

		CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
			10,        // periodic interval is 10 ms
			200000,    // timeout is very long (200 s)
			0L,        // no extra delay
			maxConcurrentAttempts,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			true,
			false,
			0);
		CheckpointCoordinator coord = new CheckpointCoordinator(
			jid,
			chkConfig,
			new ExecutionVertex[] { triggerVertex },
			new ExecutionVertex[] { ackVertex },
			new ExecutionVertex[] { commitVertex },
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(2),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY,
			failureManager);

		coord.startCheckpointScheduler();

		// after a while, there should be exactly as many checkpoints
		// as concurrently permitted
		long now = System.currentTimeMillis();
		long timeout = now + 60000;
		long minDuration = now + 100;
		do {
			Thread.sleep(20);
		}
		while ((now = System.currentTimeMillis()) < minDuration ||
				(numCalls.get() < maxConcurrentAttempts && now < timeout));

		assertEquals(maxConcurrentAttempts, numCalls.get());

		verify(triggerVertex.getCurrentExecutionAttempt(), times(maxConcurrentAttempts))
				.triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		// now, once we acknowledge one checkpoint, it should trigger the next one
		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID, 1L), TASK_MANAGER_LOCATION_INFO);

		// this should have immediately triggered a new checkpoint
		now = System.currentTimeMillis();
		timeout = now + 60000;
		do {
			Thread.sleep(20);
		}
		while (numCalls.get() < maxConcurrentAttempts + 1 && now < timeout);

		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		// no further checkpoints should happen
		Thread.sleep(200);
		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		coord.shutdown(JobStatus.FINISHED);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private void testMaxConcurrentAttempts(int maxConcurrentAttempts) {
	try {
		final JobID jid = new JobID();

		// create some mock execution vertices and trigger some checkpoint
		final ExecutionAttemptID triggerAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID ackAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID commitAttemptID = new ExecutionAttemptID();

		ExecutionVertex triggerVertex = mockExecutionVertex(triggerAttemptID);
		ExecutionVertex ackVertex = mockExecutionVertex(ackAttemptID);
		ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);

		final AtomicInteger numCalls = new AtomicInteger();

		final Execution execution = triggerVertex.getCurrentExecutionAttempt();

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		doAnswer(invocation -> {
			numCalls.incrementAndGet();
			return null;
		}).when(execution).notifyCheckpointComplete(anyLong(), anyLong());

		CheckpointCoordinatorConfiguration chkConfig =
			new CheckpointCoordinatorConfigurationBuilder()
				.setCheckpointInterval(10) // periodic interval is 10 ms
				.setCheckpointTimeout(200000) // timeout is very long (200 s)
				.setMinPauseBetweenCheckpoints(0L) // no extra delay
				.setMaxConcurrentCheckpoints(maxConcurrentAttempts)
				.build();
		CheckpointCoordinator coord =
			new CheckpointCoordinatorBuilder()
				.setJobId(jid)
				.setCheckpointCoordinatorConfiguration(chkConfig)
				.setTasksToTrigger(new ExecutionVertex[] { triggerVertex })
				.setTasksToWaitFor(new ExecutionVertex[] { ackVertex })
				.setTasksToCommitTo(new ExecutionVertex[] { commitVertex })
				.setCompletedCheckpointStore(new StandaloneCompletedCheckpointStore(2))
				.setTimer(manuallyTriggeredScheduledExecutor)
				.build();

		coord.startCheckpointScheduler();

		for (int i = 0; i < maxConcurrentAttempts; i++) {
			manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
			manuallyTriggeredScheduledExecutor.triggerAll();
		}

		assertEquals(maxConcurrentAttempts, numCalls.get());

		verify(triggerVertex.getCurrentExecutionAttempt(), times(maxConcurrentAttempts))
				.triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		// now, once we acknowledge one checkpoint, it should trigger the next one
		coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID, 1L), TASK_MANAGER_LOCATION_INFO);

		final Collection<ScheduledFuture<?>> periodicScheduledTasks =
			manuallyTriggeredScheduledExecutor.getPeriodicScheduledTask();
		assertEquals(1, periodicScheduledTasks.size());
		final ScheduledFuture scheduledFuture = periodicScheduledTasks.iterator().next();

		manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
		manuallyTriggeredScheduledExecutor.triggerAll();

		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		// no further checkpoints should happen
		manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
		manuallyTriggeredScheduledExecutor.triggerAll();
		assertEquals(maxConcurrentAttempts + 1, numCalls.get());

		coord.shutdown(JobStatus.FINISHED);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testPeriodicTriggering() {
	try {
		final JobID jid = new JobID();
		final long start = System.currentTimeMillis();

		// create some mock execution vertices and trigger some checkpoint

		final ExecutionAttemptID triggerAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID ackAttemptID = new ExecutionAttemptID();
		final ExecutionAttemptID commitAttemptID = new ExecutionAttemptID();

		ExecutionVertex triggerVertex = mockExecutionVertex(triggerAttemptID);
		ExecutionVertex ackVertex = mockExecutionVertex(ackAttemptID);
		ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);

		final AtomicInteger numCalls = new AtomicInteger();

		final Execution execution = triggerVertex.getCurrentExecutionAttempt();

		doAnswer(new Answer<Void>() {

			private long lastId = -1;
			private long lastTs = -1;

			@Override
			public Void answer(InvocationOnMock invocation) throws Throwable {
				long id = (Long) invocation.getArguments()[0];
				long ts = (Long) invocation.getArguments()[1];

				assertTrue(id > lastId);
				assertTrue(ts >= lastTs);
				assertTrue(ts >= start);

				lastId = id;
				lastTs = ts;
				numCalls.incrementAndGet();
				return null;
			}
		}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));

		CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration =
			new CheckpointCoordinatorConfigurationBuilder()
				.setCheckpointInterval(10) // periodic interval is 10 ms
				.setCheckpointTimeout(200000) // timeout is very long (200 s)
				.setMaxConcurrentCheckpoints(Integer.MAX_VALUE)
				.build();
		CheckpointCoordinator checkpointCoordinator =
			new CheckpointCoordinatorBuilder()
				.setJobId(jid)
				.setCheckpointCoordinatorConfiguration(checkpointCoordinatorConfiguration)
				.setTasksToTrigger(new ExecutionVertex[] { triggerVertex })
				.setTasksToWaitFor(new ExecutionVertex[] { ackVertex })
				.setTasksToCommitTo(new ExecutionVertex[] { commitVertex })
				.setCompletedCheckpointStore(new StandaloneCompletedCheckpointStore(2))
				.setTimer(manuallyTriggeredScheduledExecutor)
				.build();

		checkpointCoordinator.startCheckpointScheduler();

		do {
			manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
			manuallyTriggeredScheduledExecutor.triggerAll();
		}
		while (numCalls.get() < 5);
		assertEquals(5, numCalls.get());

		checkpointCoordinator.stopCheckpointScheduler();

		// no further calls may come.
		manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
		manuallyTriggeredScheduledExecutor.triggerAll();
		assertEquals(5, numCalls.get());

		// start another sequence of periodic scheduling
		numCalls.set(0);
		checkpointCoordinator.startCheckpointScheduler();

		do {
			manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
			manuallyTriggeredScheduledExecutor.triggerAll();
		}
		while (numCalls.get() < 5);
		assertEquals(5, numCalls.get());

		checkpointCoordinator.stopCheckpointScheduler();

		// no further calls may come
		manuallyTriggeredScheduledExecutor.triggerPeriodicScheduledTasks();
		manuallyTriggeredScheduledExecutor.triggerAll();
		assertEquals(5, numCalls.get());

		checkpointCoordinator.shutdown(JobStatus.FINISHED);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}