org.apache.flink.runtime.checkpoint.savepoint.Savepoint Java Examples

public static Savepoint loadCheckpointMetadata(DataInputStream in, ClassLoader classLoader) throws IOException {
	checkNotNull(in, "input stream");
	checkNotNull(classLoader, "classLoader");

	final int magicNumber = in.readInt();

	if (magicNumber == HEADER_MAGIC_NUMBER) {
		final int version = in.readInt();
		final SavepointSerializer<?> serializer = SavepointSerializers.getSerializer(version);

		if (serializer != null) {
			return serializer.deserialize(in, classLoader);
		}
		else {
			throw new IOException("Unrecognized checkpoint version number: " + version);
		}
	}
	else {
		throw new IOException("Unexpected magic number. This can have multiple reasons: " +
				"(1) You are trying to load a Flink 1.0 savepoint, which is not supported by this " +
				"version of Flink. (2) The file you were pointing to is not a savepoint at all. " +
				"(3) The savepoint file has been corrupted.");
	}
}
 

private Path validateStateAndTransform(Savepoint savepoint) throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();

	// Validate the contents of the broadcast state
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "stateful");
	OperatorStateBackend backend = reader.createOperatorStateBackendFromSnapshot(1);
	BroadcastState<Boolean, List<Integer>> state = backend.getBroadcastState(bcstate);
	assertEquals(Lists.newArrayList(1), state.get(true));

	Path newCheckpointBasePath = new Path(getCheckpointDir(), "new");
	OperatorStateWriter writer = new OperatorStateWriter(savepoint, "stateful", newCheckpointBasePath);
	writer.transformNonKeyedState((i, b) -> {
		try {
			b.getBroadcastState(bcstate).put(true, Lists.newArrayList(2, 3));
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	});

	StateMetadataUtils.writeSavepointMetadata(newCheckpointBasePath,
			StateMetadataUtils.createNewSavepoint(savepoint, writer.writeAll()));
	return newCheckpointBasePath;
}
 

private Path transformLastSavepoint() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastSavepoint();
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "hello");

	DataSet<Tuple2<Integer, String>> sumStringState = reader.readKeyedStates(
			KeyedStateReader.forValueStateKVPairs("sum", new TypeHint<Tuple2<Integer, String>>() {}));

	DataSet<Tuple2<Integer, Integer>> sumIntState = sumStringState.map(t -> Tuple2.of(t.f0, Integer.parseInt(t.f1)))
			.returns(new TypeHint<Tuple2<Integer, Integer>>() {});

	Path newCheckpointBasePath = new Path(getCheckpointDir(), "new");

	OperatorStateWriter sumStateWriter = new OperatorStateWriter(savepoint, "hello", newCheckpointBasePath);
	sumStateWriter.createNewValueState("sum", sumIntState, IntSerializer.INSTANCE);

	Savepoint newSavepoint = StateMetadataUtils.createNewSavepoint(savepoint, sumStateWriter.writeAll());
	StateMetadataUtils.writeSavepointMetadata(newCheckpointBasePath, newSavepoint);
	return newCheckpointBasePath;
}
 

private Path transformLastSavepoint() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastSavepoint();

	DataSet<Tuple2<Integer, Integer>> bootstrapState = environment.fromElements(Tuple2.of(1, 100),
			Tuple2.of(2, 100));

	Path newCheckpointBasePath = new Path(getCheckpointDir(), "new");

	OperatorStateWriter counterStateWriter = new OperatorStateWriter(savepoint, "counter", newCheckpointBasePath);

	counterStateWriter.setKeySerializer(IntSerializer.INSTANCE);
	counterStateWriter.createNewValueState("count", bootstrapState, IntSerializer.INSTANCE);

	OperatorStateWriter filterStateWriter = new OperatorStateWriter(savepoint, "filter", newCheckpointBasePath);
	filterStateWriter.deleteKeyedState("seen");

	Savepoint newSavepoint = StateMetadataUtils.createNewSavepoint(savepoint,
			filterStateWriter.writeAll(),
			counterStateWriter.writeAll());
	StateMetadataUtils.writeSavepointMetadata(newCheckpointBasePath, newSavepoint);
	return newCheckpointBasePath;
}
 

@SuppressWarnings("unchecked")
private void validateCheckpointedStateReading() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastCheckpoint();
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "hello");

	List<Tuple3<Integer, String, Integer>> countState = reader
			.readKeyedStates(KeyedStateReader.forMapStateEntries("Count", BasicTypeInfo.INT_TYPE_INFO,
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	List<Integer> mapValues = reader
			.readKeyedStates(KeyedStateReader.forMapStateValues("Count", BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	assertEquals(Sets.newHashSet(Tuple3.of(1, "1", 2), Tuple3.of(1, "2", 1), Tuple3.of(2, "3", 1)),
			new HashSet<>(countState));

	Collections.sort(mapValues);
	assertEquals(Lists.newArrayList(1, 1, 2), mapValues);
}
 

public static Savepoint loadCheckpointMetadata(DataInputStream in, ClassLoader classLoader) throws IOException {
	checkNotNull(in, "input stream");
	checkNotNull(classLoader, "classLoader");

	final int magicNumber = in.readInt();

	if (magicNumber == HEADER_MAGIC_NUMBER) {
		final int version = in.readInt();
		final SavepointSerializer<?> serializer = SavepointSerializers.getSerializer(version);

		if (serializer != null) {
			return serializer.deserialize(in, classLoader);
		}
		else {
			throw new IOException("Unrecognized checkpoint version number: " + version);
		}
	}
	else {
		throw new IOException("Unexpected magic number. This can have multiple reasons: " +
				"(1) You are trying to load a Flink 1.0 savepoint, which is not supported by this " +
				"version of Flink. (2) The file you were pointing to is not a savepoint at all. " +
				"(3) The savepoint file has been corrupted.");
	}
}
 

/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 

public static <T extends Savepoint> void storeCheckpointMetadata(
		T checkpointMetadata,
		DataOutputStream out) throws IOException {

	// write generic header
	out.writeInt(HEADER_MAGIC_NUMBER);
	out.writeInt(checkpointMetadata.getVersion());

	// write checkpoint metadata
	SavepointSerializer<T> serializer = SavepointSerializers.getSerializer(checkpointMetadata);
	serializer.serialize(checkpointMetadata, out);
}
 

public static void main(String[] args) throws IOException {
    //  读取元数据文件
    File f = new File("flink-learning-state/src/main/resources/_metadata");
    // 第二步，建立管道,FileInputStream文件输入流类用于读文件
    FileInputStream fis = new FileInputStream(f);
    BufferedInputStream bis = new BufferedInputStream(fis);
    DataInputStream dis = new DataInputStream(bis);

    // 通过 Flink 的 Checkpoints 类解析元数据文件
    Savepoint savepoint = Checkpoints.loadCheckpointMetadata(dis,
            MetadataSerializer.class.getClassLoader());
    // 打印当前的 CheckpointId
    System.out.println(savepoint.getCheckpointId());

    // 遍历 OperatorState，这里的每个 OperatorState 对应一个 Flink 任务的 Operator 算子
    // 不要与 OperatorState  和 KeyedState 混淆，不是一个层级的概念
    for (OperatorState operatorState : savepoint.getOperatorStates()) {
        System.out.println(operatorState);
        // 当前算子的状态大小为 0 ，表示算子不带状态，直接退出
        if (operatorState.getStateSize() == 0) {
            continue;
        }

        // 遍历当前算子的所有 subtask
        for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
            // 解析 operatorSubtaskState 的 ManagedKeyedState
            parseManagedKeyedState(operatorSubtaskState);
            // 解析 operatorSubtaskState 的 ManagedOperatorState
            parseManagedOperatorState(operatorSubtaskState);
        }
    }
}
 

private Path transformLastSavepoint() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastSavepoint();
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "hello");

	DataSet<Tuple2<Integer, Integer>> countState = reader.readKeyedStates(
			KeyedStateReader.forValueStateKVPairs("Count", new TypeHint<Tuple2<Integer, Integer>>() {}));

	DataSet<Tuple2<Integer, Integer>> newCountsToAdd = environment
			.fromElements(Tuple2.of(0, 100), Tuple2.of(3, 1000), Tuple2.of(1, 100), Tuple2.of(2, 1000));

	DataSet<Tuple2<Integer, Integer>> newStates = countState.join(newCountsToAdd).where(0).equalTo(0)
			.map(new SumValues());

	Path newCheckpointBasePath = new Path(getCheckpointDir(), "new");
	OperatorStateWriter operatorStateWriter = new OperatorStateWriter(savepoint, "hello", newCheckpointBasePath);

	operatorStateWriter.addValueState("Count",
			countState.map(t -> Tuple2.of(t.f0, t.f1 * 2)).returns(new TypeHint<Tuple2<Integer, Integer>>() {}));

	operatorStateWriter.createNewValueState("Count2", newStates, IntSerializer.INSTANCE);
	operatorStateWriter.addKeyedStateRows(reader.getAllUnreadKeyedStateRows());

	OperatorState newOpState = operatorStateWriter.writeAll();
	Savepoint newSavepoint = StateMetadataUtils.createNewSavepoint(savepoint, newOpState);
	StateMetadataUtils.writeSavepointMetadata(newCheckpointBasePath, newSavepoint);
	return newCheckpointBasePath;
}
 

private void validateSecondSavepoint(Savepoint savepoint) throws Exception {
	// Validate the contents of the broadcast state
	OperatorStateReader reader = new OperatorStateReader(ExecutionEnvironment.createLocalEnvironment(), savepoint,
			"stateful");
	OperatorStateBackend backend = reader.createOperatorStateBackendFromSnapshot(1);
	BroadcastState<Boolean, List<Integer>> state = backend.getBroadcastState(bcstate);
	assertEquals(Lists.newArrayList(2, 3), state.get(true));
}
 

@Test
public void test() throws Exception {
	process("process 1");
	sleep(500);
	process("filter 1");
	sleep(1000);
	process("process 1");
	process("process 2");
	process("process 3");
	sleep(1000);
	cancelJob();
	List<String> output = runTestPipeline(this::constructTestPipeline);
	assertEquals(3, output.size());

	process("process 1");
	process("process 2");
	process("process 3");
	triggerSavepoint();

	System.err.println("Round two");
	List<String> output2 = restoreTestPipelineFromSnapshot(
			validateStateAndTransform(getLastCheckpoint()).getPath(),
			this::constructTestPipeline);

	Savepoint savepoint = getLastSavepoint();

	validateSecondSavepoint(savepoint);
	assertEquals(4, output2.size());
}
 

@SuppressWarnings("unchecked")
private void validateCheckpointedStateReading() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastCheckpoint();
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "hello");

	DataSet<Tuple2<Integer, Integer>> countState = reader.readKeyedStates(
			KeyedStateReader.forValueStateKVPairs("Count", new TypeHint<Tuple2<Integer, Integer>>() {}));

	assertEquals(Sets.newHashSet(Tuple2.of(1, 2), Tuple2.of(2, 1)), new HashSet<>(countState.collect()));
}
 

public OperatorStateReader(ExecutionEnvironment env, Savepoint sp, String uid, Collection<String> stateNames) {
	this(env, sp, uid, new FilterFunction<String>() {
		private static final long serialVersionUID = 1L;
		HashSet<String> filtered = new HashSet<>(stateNames);

		@Override
		public boolean filter(String s) throws Exception {
			return filtered.contains(s);
		}
	});
}
 

public static OperatorState getOperatorState(Savepoint savepoint, OperatorID opId) {
	return savepoint
			.getOperatorStates()
			.stream()
			.filter(os -> os.getOperatorID().equals(opId))
			.findAny()
			.orElseThrow(() -> new RuntimeException("No operator state with id " + opId.toString()));
}
 

public static <T extends Savepoint> void storeCheckpointMetadata(
		T checkpointMetadata,
		DataOutputStream out) throws IOException {

	// write generic header
	out.writeInt(HEADER_MAGIC_NUMBER);
	out.writeInt(checkpointMetadata.getVersion());

	// write checkpoint metadata
	SavepointSerializer<T> serializer = SavepointSerializers.getSerializer(checkpointMetadata);
	serializer.serialize(checkpointMetadata, out);
}
 

public static <T extends Savepoint> void storeCheckpointMetadata(
		T checkpointMetadata,
		OutputStream out) throws IOException {

	DataOutputStream dos = new DataOutputStream(out);
	storeCheckpointMetadata(checkpointMetadata, dos);
}
 

public static void main(String[] args) throws IOException {
    //  读取元数据文件
    File f = new File("flink-learning-state/src/main/resources/_metadata");
    // 第二步，建立管道,FileInputStream文件输入流类用于读文件
    FileInputStream fis = new FileInputStream(f);
    BufferedInputStream bis = new BufferedInputStream(fis);
    DataInputStream dis = new DataInputStream(bis);

    // 通过 Flink 的 Checkpoints 类解析元数据文件
    Savepoint savepoint = Checkpoints.loadCheckpointMetadata(dis,
            MetadataSerializer.class.getClassLoader());
    // 打印当前的 CheckpointId
    System.out.println(savepoint.getCheckpointId());

    // 遍历 OperatorState，这里的每个 OperatorState 对应一个 Flink 任务的 Operator 算子
    // 不要与 OperatorState  和 KeyedState 混淆，不是一个层级的概念
    for (OperatorState operatorState : savepoint.getOperatorStates()) {
        System.out.println(operatorState);
        // 当前算子的状态大小为 0 ，表示算子不带状态，直接退出
        if (operatorState.getStateSize() == 0) {
            continue;
        }

        // 遍历当前算子的所有 subtask
        for (OperatorSubtaskState operatorSubtaskState : operatorState.getStates()) {
            // 解析 operatorSubtaskState 的 ManagedKeyedState
            parseManagedKeyedState(operatorSubtaskState);
            // 解析 operatorSubtaskState 的 ManagedOperatorState
            parseManagedOperatorState(operatorSubtaskState);
        }
    }
}
 

@Override
public void writeRecord(Savepoint savepoint) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(savepoint, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

@Override
public void reduce(Iterable<OperatorState> values, Collector<Savepoint> out) {
	Savepoint savepoint =
		new SavepointV2(
			SnapshotUtils.CHECKPOINT_ID,
			StreamSupport.stream(values.spliterator(), false).collect(Collectors.toList()),
			masterStates);

	out.collect(savepoint);
}
 

@Test
public void testSavepointOutputFormat() throws Exception {
	Path path = new Path(temporaryFolder.newFolder().getAbsolutePath());
	SavepointOutputFormat format = createSavepointOutputFormat(path);

	Savepoint savepoint = createSavepoint();

	format.open(0, 1);
	format.writeRecord(savepoint);
	format.close();

	Savepoint savepointOnDisk = SavepointLoader.loadSavepoint(path.getPath());

	Assert.assertEquals(
		"Incorrect checkpoint id",
		savepoint.getCheckpointId(),
		savepointOnDisk.getCheckpointId());

	Assert.assertEquals(
		"Incorrect number of operator states in savepoint",
		savepoint.getOperatorStates().size(),
		savepointOnDisk.getOperatorStates().size());

	Assert.assertEquals(
		"Incorrect operator state in savepoint",
		savepoint.getOperatorStates().iterator().next(),
		savepointOnDisk.getOperatorStates().iterator().next());
}
 

public static <T extends Savepoint> void storeCheckpointMetadata(
		T checkpointMetadata,
		OutputStream out) throws IOException {

	DataOutputStream dos = new DataOutputStream(out);
	storeCheckpointMetadata(checkpointMetadata, dos);
}
 

public OperatorStateReader(ExecutionEnvironment env, Savepoint sp, String uid) {
	this(env, sp, uid, s -> true);
}
 

public CompletedCheckpoint finalizeCheckpoint() throws IOException {

		synchronized (lock) {
			checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet.");

			// make sure we fulfill the promise with an exception if something fails
			try {
				// write out the metadata
				final Savepoint savepoint = new SavepointV2(checkpointId, operatorStates.values(), masterState);
				final CompletedCheckpointStorageLocation finalizedLocation;

				try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
					Checkpoints.storeCheckpointMetadata(savepoint, out);
					finalizedLocation = out.closeAndFinalizeCheckpoint();
				}

				CompletedCheckpoint completed = new CompletedCheckpoint(
						jobId,
						checkpointId,
						checkpointTimestamp,
						System.currentTimeMillis(),
						operatorStates,
						masterState,
						props,
						finalizedLocation);

				onCompletionPromise.complete(completed);

				// to prevent null-pointers from concurrent modification, copy reference onto stack
				PendingCheckpointStats statsCallback = this.statsCallback;
				if (statsCallback != null) {
					// Finalize the statsCallback and give the completed checkpoint a
					// callback for discards.
					CompletedCheckpointStats.DiscardCallback discardCallback =
							statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
					completed.setDiscardCallback(discardCallback);
				}

				// mark this pending checkpoint as disposed, but do NOT drop the state
				dispose(false);

				return completed;
			}
			catch (Throwable t) {
				onCompletionPromise.completeExceptionally(t);
				ExceptionUtils.rethrowIOException(t);
				return null; // silence the compiler
			}
		}
	}
 

public OperatorStateReader(ExecutionEnvironment env, Savepoint sp, String uid,
		FilterFunction<String> keyedStateFilter) {
	this(env, StateMetadataUtils.getOperatorState(sp, uid), keyedStateFilter);
}
 

public OperatorStateWriter(Savepoint sp, String uid, Path newCheckpointBasePath) {
	this(sp.getCheckpointId(), StateMetadataUtils.getOperatorState(sp, uid), newCheckpointBasePath);
}
 

public static Path writeSavepointMetadata(Path newCheckpointBasePath, Savepoint savepoint) throws IOException {
	Path p = new Path(newCheckpointBasePath, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	Checkpoints.storeCheckpointMetadata(savepoint,
			newCheckpointBasePath.getFileSystem().create(p, WriteMode.NO_OVERWRITE));
	return p;
}
 

@SuppressWarnings("unchecked")
private void validateCheckpointedStateReading() throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();
	Savepoint savepoint = getLastCheckpoint();
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "hello");

	List<Tuple3<Integer, String, Integer>> countState = reader
			.readKeyedStates(KeyedStateReader.forMapStateEntries("Map", BasicTypeInfo.INT_TYPE_INFO,
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	List<Integer> mapValues = reader
			.readKeyedStates(KeyedStateReader.forMapStateValues("Map", BasicTypeInfo.INT_TYPE_INFO, true))
			.collect();

	Collections.sort(mapValues);
	assertEquals(Lists.newArrayList(1, 1, 2), mapValues);

	assertEquals(Sets.newHashSet(Tuple3.of(1, "1", 2), Tuple3.of(1, "2", 1), Tuple3.of(2, "3", 1)),
			new HashSet<>(countState));

	List<Tuple2<Integer, List<Integer>>> listState = reader.readKeyedStates(
			KeyedStateReader.forListStates("List", BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	assertEquals(Sets.newHashSet(Tuple2.of(1, Lists.newArrayList(1, 2, 1)), Tuple2.of(2, Lists.newArrayList(3))),
			new HashSet<>(listState));

	List<Tuple2<Integer, Integer>> listStateValues = reader.readKeyedStates(
			KeyedStateReader.forListStateValues("List", BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	assertEquals(Sets.newHashSet(Tuple2.of(1, 1), Tuple2.of(1, 2), Tuple2.of(2, 3)),
			new HashSet<>(listStateValues));

	List<Tuple2<Integer, Integer>> valuePairs = reader.readKeyedStates(
			KeyedStateReader.forValueStateKVPairs("Val", BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	assertEquals(Sets.newHashSet(Tuple2.of(1, 1), Tuple2.of(2, 3)), new HashSet<>(valuePairs));

	List<Integer> values = reader.readKeyedStates(
			KeyedStateReader.forValueStateValues("Val", BasicTypeInfo.INT_TYPE_INFO))
			.collect();

	assertEquals(Sets.newHashSet(1, 3), new HashSet<>(values));

}
 

protected Savepoint getLastCheckpoint() throws IOException {
	return loadSavepoint(getLastCheckpointPath().getPath());
}