Java Code Examples for org.apache.flink.runtime.state.CompletedCheckpointStorageLocation#getMetadataHandle()

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

public static CompletedCheckpoint loadAndValidateCheckpoint(
		JobID jobId,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		CompletedCheckpointStorageLocation location,
		ClassLoader classLoader,
		boolean allowNonRestoredState) throws IOException {

	checkNotNull(jobId, "jobId");
	checkNotNull(tasks, "tasks");
	checkNotNull(location, "location");
	checkNotNull(classLoader, "classLoader");

	final StreamStateHandle metadataHandle = location.getMetadataHandle();
	final String checkpointPointer = location.getExternalPointer();

	// (1) load the savepoint
	final CheckpointMetadata checkpointMetadata;
	try (InputStream in = metadataHandle.openInputStream()) {
		DataInputStream dis = new DataInputStream(in);
		checkpointMetadata = loadCheckpointMetadata(dis, classLoader, checkpointPointer);
	}

	// generate mapping from operator to task
	Map<OperatorID, ExecutionJobVertex> operatorToJobVertexMapping = new HashMap<>();
	for (ExecutionJobVertex task : tasks.values()) {
		for (OperatorIDPair operatorIDPair : task.getOperatorIDs()) {
			operatorToJobVertexMapping.put(operatorIDPair.getGeneratedOperatorID(), task);
			operatorIDPair.getUserDefinedOperatorID().ifPresent(id -> operatorToJobVertexMapping.put(id, task));
		}
	}

	// (2) validate it (parallelism, etc)
	HashMap<OperatorID, OperatorState> operatorStates = new HashMap<>(checkpointMetadata.getOperatorStates().size());
	for (OperatorState operatorState : checkpointMetadata.getOperatorStates()) {

		ExecutionJobVertex executionJobVertex = operatorToJobVertexMapping.get(operatorState.getOperatorID());

		if (executionJobVertex != null) {

			if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism()
					|| !executionJobVertex.isMaxParallelismConfigured()) {
				operatorStates.put(operatorState.getOperatorID(), operatorState);
			} else {
				String msg = String.format("Failed to rollback to checkpoint/savepoint %s. " +
								"Max parallelism mismatch between checkpoint/savepoint state and new program. " +
								"Cannot map operator %s with max parallelism %d to new program with " +
								"max parallelism %d. This indicates that the program has been changed " +
								"in a non-compatible way after the checkpoint/savepoint.",
						checkpointMetadata,
						operatorState.getOperatorID(),
						operatorState.getMaxParallelism(),
						executionJobVertex.getMaxParallelism());

				throw new IllegalStateException(msg);
			}
		} else if (allowNonRestoredState) {
			LOG.info("Skipping savepoint state for operator {}.", operatorState.getOperatorID());
		} else {
			if (operatorState.getCoordinatorState() != null) {
				throwNonRestoredStateException(checkpointPointer, operatorState.getOperatorID());
			}

			for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
				if (operatorSubtaskState.hasState()) {
					throwNonRestoredStateException(checkpointPointer, operatorState.getOperatorID());
				}
			}

			LOG.info("Skipping empty savepoint state for operator {}.", operatorState.getOperatorID());
		}
	}

	// (3) convert to checkpoint so the system can fall back to it
	CheckpointProperties props = CheckpointProperties.forSavepoint(false);

	return new CompletedCheckpoint(
			jobId,
			checkpointMetadata.getCheckpointId(),
			0L,
			0L,
			operatorStates,
			checkpointMetadata.getMasterStates(),
			props,
			location);
}