org.apache.flink.runtime.state.CompletedCheckpointStorageLocation Java Examples

/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

/**
 * Tests that savepoint loading fails when there is non-restored state, but it is not allowed.
 */
@Test
public void testNonRestoredStateWhenDisallowed() throws Exception {
	final OperatorID operatorId = new OperatorID();
	final int parallelism = 9;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(242L, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = Collections.emptyMap();

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("allowNonRestoredState"));
	}
}
 

/**
 * Tests that savepoint loading fails when there is non-restored coordinator state only,
 * and non-restored state is not allowed.
 */
@Test
public void testUnmatchedCoordinatorOnlyStateFails() throws Exception {
	final OperatorID operatorID = new OperatorID();
	final int maxParallelism = 1234;

	final OperatorState state = new OperatorState(operatorID, maxParallelism / 2, maxParallelism);
	state.setCoordinatorState(new ByteStreamStateHandle("coordinatorState", new byte[0]));

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorState(42L, state);
	final Map<JobVertexID, ExecutionJobVertex> tasks = Collections.emptyMap();

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("allowNonRestoredState"));
	}
}
 

private static CompletedCheckpointStorageLocation createSavepointWithOperatorSubtaskState(
		final long checkpointId,
		final OperatorID operatorId,
		final int parallelism) throws IOException {

	final Random rnd = new Random();

	final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
		new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("testHandler", new byte[0])),
		null,
		null,
		null,
		singleton(createNewInputChannelStateHandle(10, rnd)),
		singleton(createNewResultSubpartitionStateHandle(10, rnd)));

	final OperatorState state = new OperatorState(operatorId, parallelism, parallelism);
	state.putState(0, subtaskState);

	return createSavepointWithOperatorState(checkpointId, state);
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

/**
 * Tests that savepoint loading fails when there is a max-parallelism mismatch.
 */
@Test
public void testMaxParallelismMismatch() throws Exception {
	final OperatorID operatorId = new OperatorID();
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(242L, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism + 1);

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("Max parallelism mismatch"));
	}
}
 

/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
	final JobID jobId = new JobID();
	final OperatorID operatorId = new OperatorID();
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);

	final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false);

	assertEquals(jobId, loaded.getJobId());
	assertEquals(checkpointId, loaded.getCheckpointID());
}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}
 

/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

@Override
public void writeRecord(Savepoint savepoint) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(savepoint, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(metadata, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
}
 

/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedStateInternal(new HashSet<>(tasks.values()), true, true, allowNonRestored);
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}
 

@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}