org.apache.flink.runtime.state.filesystem.AbstractFsCheckpointStorage Java Examples

public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 

private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 

public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(
	OP operator,
	int index,
	long timestamp,
	CheckpointStorageWorkerView checkpointStorage,
	Path savepointPath) throws Exception {

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.SAVEPOINT,
		AbstractFsCheckpointStorage.encodePathAsReference(savepointPath));

	operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);

	CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation(
		CHECKPOINT_ID,
		options.getTargetLocation());

	OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(
		CHECKPOINT_ID,
		timestamp,
		options,
		storage);

	OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();

	operator.notifyCheckpointComplete(CHECKPOINT_ID);
	return new TaggedOperatorSubtaskState(index, state);
}
 

/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 

private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		(int) CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue().getBytes(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 

public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(
	OP operator,
	int index,
	long timestamp,
	boolean isExactlyOnceMode,
	boolean isUnalignedCheckpoint,
	CheckpointStorageWorkerView checkpointStorage,
	Path savepointPath) throws Exception {

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.SAVEPOINT,
		AbstractFsCheckpointStorage.encodePathAsReference(savepointPath),
		isExactlyOnceMode,
		isUnalignedCheckpoint);

	operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);

	CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation(
		CHECKPOINT_ID,
		options.getTargetLocation());

	OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(
		CHECKPOINT_ID,
		timestamp,
		options,
		storage);

	OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();

	operator.notifyCheckpointComplete(CHECKPOINT_ID);
	return new TaggedOperatorSubtaskState(index, state);
}
 

private static Path createExclusiveDirPath(String externalPointer) throws IOException {
	try {
		return AbstractFsCheckpointStorage.resolveCheckpointPointer(externalPointer).getExclusiveCheckpointDir();
	} catch (IOException e) {
		throw new IOException("Could not parse external pointer as state base path", e);
	}
}
 

/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 

public static Path writeSavepointMetadata(Path newCheckpointBasePath, Savepoint savepoint) throws IOException {
	Path p = new Path(newCheckpointBasePath, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	Checkpoints.storeCheckpointMetadata(savepoint,
			newCheckpointBasePath.getFileSystem().create(p, WriteMode.NO_OVERWRITE));
	return p;
}
 

/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static Savepoint loadSavepoint(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader());
	}
}
 

/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static CheckpointMetadata loadSavepointMetadata(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader(), savepointPath);
	}
}