org.apache.flink.runtime.checkpoint.Checkpoints Java Examples

@Override
public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath, Time timeout) {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();

	return CompletableFuture.supplyAsync(
		() -> {
			log.info("Disposing savepoint {}.", savepointPath);

			try {
				Checkpoints.disposeSavepoint(savepointPath, configuration, classLoader, log);
			} catch (IOException | FlinkException e) {
				throw new CompletionException(new FlinkException(String.format("Could not dispose savepoint %s.", savepointPath), e));
			}

			return Acknowledge.get();
		},
		jobManagerSharedServices.getScheduledExecutorService());
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Override
public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath, Time timeout) {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();

	return CompletableFuture.supplyAsync(
		() -> {
			log.info("Disposing savepoint {}.", savepointPath);

			try {
				Checkpoints.disposeSavepoint(savepointPath, configuration, classLoader, log);
			} catch (IOException | FlinkException e) {
				throw new CompletionException(new FlinkException(String.format("Could not dispose savepoint %s.", savepointPath), e));
			}

			return Acknowledge.get();
		},
		jobManagerSharedServices.getScheduledExecutorService());
}
 

@Override
public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath, Time timeout) {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();

	return CompletableFuture.supplyAsync(
		() -> {
			log.info("Disposing savepoint {}.", savepointPath);

			try {
				Checkpoints.disposeSavepoint(savepointPath, configuration, classLoader, log);
			} catch (IOException | FlinkException e) {
				throw new CompletionException(new FlinkException(String.format("Could not dispose savepoint %s.", savepointPath), e));
			}

			return Acknowledge.get();
		},
		jobManagerSharedServices.getScheduledExecutorService());
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

private static byte[] serializeAsCheckpointMetadata(OperatorID id, byte[] coordinatorState) throws IOException {
	final OperatorState state = createOperatorState(id, coordinatorState);
	final CheckpointMetadata metadata = new CheckpointMetadata(
		1337L, Collections.singletonList(state), Collections.emptyList());

	final ByteArrayOutputStream out = new ByteArrayOutputStream();
	Checkpoints.storeCheckpointMetadata(metadata, out);
	return out.toByteArray();
}
 

private File createSavepointWithOperatorState(long savepointId, OperatorID... operatorIds) throws IOException {
	final File savepointFile = temporaryFolder.newFile();
	final Collection<OperatorState> operatorStates = createOperatorState(operatorIds);
	final CheckpointMetadata savepoint = new CheckpointMetadata(savepointId, operatorStates, Collections.emptyList());

	try (FileOutputStream fileOutputStream = new FileOutputStream(savepointFile)) {
		Checkpoints.storeCheckpointMetadata(savepoint, fileOutputStream);
	}

	return savepointFile;
}
 

@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(metadata, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

public static void main(String[] args) throws IOException {
    //  读取元数据文件
    File f = new File("flink-learning-state/src/main/resources/_metadata");
    // 第二步，建立管道,FileInputStream文件输入流类用于读文件
    FileInputStream fis = new FileInputStream(f);
    BufferedInputStream bis = new BufferedInputStream(fis);
    DataInputStream dis = new DataInputStream(bis);

    // 通过 Flink 的 Checkpoints 类解析元数据文件
    Savepoint savepoint = Checkpoints.loadCheckpointMetadata(dis,
            MetadataSerializer.class.getClassLoader());
    // 打印当前的 CheckpointId
    System.out.println(savepoint.getCheckpointId());

    // 遍历 OperatorState，这里的每个 OperatorState 对应一个 Flink 任务的 Operator 算子
    // 不要与 OperatorState  和 KeyedState 混淆，不是一个层级的概念
    for (OperatorState operatorState : savepoint.getOperatorStates()) {
        System.out.println(operatorState);
        // 当前算子的状态大小为 0 ，表示算子不带状态，直接退出
        if (operatorState.getStateSize() == 0) {
            continue;
        }

        // 遍历当前算子的所有 subtask
        for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
            // 解析 operatorSubtaskState 的 ManagedKeyedState
            parseManagedKeyedState(operatorSubtaskState);
            // 解析 operatorSubtaskState 的 ManagedOperatorState
            parseManagedOperatorState(operatorSubtaskState);
        }
    }
}
 

private File createSavepointWithOperatorState(long savepointId, OperatorID... operatorIds) throws IOException {
	final File savepointFile = temporaryFolder.newFile();
	final Collection<OperatorState> operatorStates = createOperatorState(operatorIds);
	final SavepointV2 savepoint = new SavepointV2(savepointId, operatorStates, Collections.emptyList());

	try (FileOutputStream fileOutputStream = new FileOutputStream(savepointFile)) {
		Checkpoints.storeCheckpointMetadata(savepoint, fileOutputStream);
	}

	return savepointFile;
}
 

@Override
public void writeRecord(Savepoint savepoint) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(savepoint, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

public static void main(String[] args) throws IOException {
    //  读取元数据文件
    File f = new File("flink-learning-state/src/main/resources/_metadata");
    // 第二步，建立管道,FileInputStream文件输入流类用于读文件
    FileInputStream fis = new FileInputStream(f);
    BufferedInputStream bis = new BufferedInputStream(fis);
    DataInputStream dis = new DataInputStream(bis);

    // 通过 Flink 的 Checkpoints 类解析元数据文件
    Savepoint savepoint = Checkpoints.loadCheckpointMetadata(dis,
            MetadataSerializer.class.getClassLoader());
    // 打印当前的 CheckpointId
    System.out.println(savepoint.getCheckpointId());

    // 遍历 OperatorState，这里的每个 OperatorState 对应一个 Flink 任务的 Operator 算子
    // 不要与 OperatorState  和 KeyedState 混淆，不是一个层级的概念
    for (OperatorState operatorState : savepoint.getOperatorStates()) {
        System.out.println(operatorState);
        // 当前算子的状态大小为 0 ，表示算子不带状态，直接退出
        if (operatorState.getStateSize() == 0) {
            continue;
        }

        // 遍历当前算子的所有 subtask
        for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
            // 解析 operatorSubtaskState 的 ManagedKeyedState
            parseManagedKeyedState(operatorSubtaskState);
            // 解析 operatorSubtaskState 的 ManagedOperatorState
            parseManagedOperatorState(operatorSubtaskState);
        }
    }
}
 

private File createSavepointWithOperatorState(long savepointId, OperatorID... operatorIds) throws IOException {
	final File savepointFile = temporaryFolder.newFile();
	final Collection<OperatorState> operatorStates = createOperatorState(operatorIds);
	final SavepointV2 savepoint = new SavepointV2(savepointId, operatorStates, Collections.emptyList());

	try (FileOutputStream fileOutputStream = new FileOutputStream(savepointFile)) {
		Checkpoints.storeCheckpointMetadata(savepoint, fileOutputStream);
	}

	return savepointFile;
}
 

/**
 * Dispose the savepoint stored under the given path.
 *
 * @param savepointPath path where the savepoint is stored
 */
private void disposeSavepoint(String savepointPath) {
	try {
		// delete the temporary savepoint
		Checkpoints.disposeSavepoint(
			savepointPath,
			jobMasterConfiguration.getConfiguration(),
			userCodeLoader,
			log);
	} catch (FlinkException | IOException e) {
		log.info("Could not dispose temporary rescaling savepoint under {}.", savepointPath, e);
	}
}
 

public static Path writeSavepointMetadata(Path newCheckpointBasePath, Savepoint savepoint) throws IOException {
	Path p = new Path(newCheckpointBasePath, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	Checkpoints.storeCheckpointMetadata(savepoint,
			newCheckpointBasePath.getFileSystem().create(p, WriteMode.NO_OVERWRITE));
	return p;
}
 

/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static Savepoint loadSavepoint(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader());
	}
}
 

/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static CheckpointMetadata loadSavepointMetadata(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader(), savepointPath);
	}
}