org.apache.flink.runtime.state.CheckpointStorage Java Examples

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	storage.initializeBaseLocations();
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	checkNotNull(jobId, "jobId");
	return new FsCheckpointStorage(
		getCheckpointPath(),
		getSavepointPath(),
		jobId,
		getMinFileSizeThreshold(),
		getWriteBufferSize());
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	checkNotNull(jobId, "jobId");
	return new FsCheckpointStorage(
		getCheckpointPath(),
		getSavepointPath(),
		jobId,
		getMinFileSizeThreshold(),
		getWriteBufferSize());
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testNoSavepointPathConfiguredNoTarget() throws Exception {
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	try {
		storage.initializeLocationForSavepoint(1337, null);
		fail("this should fail with an exception");
	}
	catch (IllegalArgumentException ignored) {}
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

public MockStreamTask(
	Environment environment,
	String name,
	Object checkpointLock,
	StreamConfig config,
	ExecutionConfig executionConfig,
	StreamTaskStateInitializer streamTaskStateInitializer,
	CloseableRegistry closableRegistry,
	StreamStatusMaintainer streamStatusMaintainer,
	CheckpointStorage checkpointStorage,
	ProcessingTimeService processingTimeService,
	BiConsumer<String, Throwable> handleAsyncException,
	Map<String, Accumulator<?, ?>> accumulatorMap
) {
	super(environment);
	this.name = name;
	this.checkpointLock = checkpointLock;
	this.config = config;
	this.executionConfig = executionConfig;
	this.streamTaskStateInitializer = streamTaskStateInitializer;
	this.closableRegistry = closableRegistry;
	this.streamStatusMaintainer = streamStatusMaintainer;
	this.checkpointStorage = checkpointStorage;
	this.processingTimeService = processingTimeService;
	this.handleAsyncException = handleAsyncException;
	this.accumulatorMap = accumulatorMap;
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Test
public void testNoSavepointPathConfiguredNoTarget() throws Exception {
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	try {
		storage.initializeLocationForSavepoint(1337, null);
		fail("this should fail with an exception");
	}
	catch (IllegalArgumentException ignored) {}
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testNoSavepointPathConfiguredNoTarget() throws Exception {
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	try {
		storage.initializeLocationForSavepoint(1337, null);
		fail("this should fail with an exception");
	}
	catch (IllegalArgumentException ignored) {}
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) {
	return new TestCheckpointStorage();
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return backend.createCheckpointStorage(jobId);
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return spy(delegate.createCheckpointStorage(jobId));
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return new MemoryBackendCheckpointStorage(jobId, null, null, 1_000_000);
}
 

protected abstract CheckpointStorage createCheckpointStorageWithSavepointDir(
Path checkpointDir,
Path savepointDir) throws Exception;
 

@Override
protected CheckpointStorage createCheckpointStorageWithSavepointDir(Path checkpointDir, Path savepointDir) throws Exception {
	return new MemoryBackendCheckpointStorage(new JobID(), checkpointDir, savepointDir, DEFAULT_MAX_STATE_SIZE);
}
 

public MockStreamTaskBuilder setCheckpointStorage(CheckpointStorage checkpointStorage) {
	this.checkpointStorage = checkpointStorage;
	return this;
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return new MemoryBackendCheckpointStorage(jobId, null, null, Integer.MAX_VALUE);
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return spy(delegate.createCheckpointStorage(jobId));
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException {
	return backend.createCheckpointStorage(jobId);
}
 

@Override
public CheckpointStorage createCheckpointStorage(JobID jobId) {
	return new TestCheckpointStorage();
}
 

@Override
protected CheckpointStorage createCheckpointStorageWithSavepointDir(Path checkpointDir, Path savepointDir) throws Exception {
	return new MemoryBackendCheckpointStorage(new JobID(), checkpointDir, savepointDir, DEFAULT_MAX_STATE_SIZE);
}
 

@Override
protected CheckpointStorage createCheckpointStorage(Path checkpointDir) throws Exception {
	return new MemoryBackendCheckpointStorage(new JobID(), checkpointDir, null, DEFAULT_MAX_STATE_SIZE);
}
 

@Override
protected CheckpointStorage createCheckpointStorageWithSavepointDir(Path checkpointDir, Path savepointDir) throws Exception {
	return new FsCheckpointStorage(checkpointDir, savepointDir, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);
}