org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory Java Examples

The following examples show how to use org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ChannelStateCheckpointWriterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
@SuppressWarnings("ConstantConditions")
public void testSmallFilesNotWritten() throws Exception {
	int threshold = 100;
	File checkpointsDir = temporaryFolder.newFolder("checkpointsDir");
	File sharedStateDir = temporaryFolder.newFolder("sharedStateDir");
	FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(getSharedInstance(), fromLocalFile(checkpointsDir), fromLocalFile(sharedStateDir), threshold, threshold);
	ChannelStateWriteResult result = new ChannelStateWriteResult();
	ChannelStateCheckpointWriter writer = createWriter(result, checkpointStreamFactory.createCheckpointStateOutputStream(EXCLUSIVE));
	NetworkBuffer buffer = new NetworkBuffer(HeapMemorySegment.FACTORY.allocateUnpooledSegment(threshold / 2, null), FreeingBufferRecycler.INSTANCE);
	writer.writeInput(new InputChannelInfo(1, 2), buffer);
	writer.completeOutput();
	writer.completeInput();
	assertTrue(result.isDone());
	assertEquals(0, checkpointsDir.list().length);
	assertEquals(0, sharedStateDir.list().length);
}
Example #2
Source File: RocksDBStateUploaderTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
	File checkpointPrivateFolder = temporaryFolder.newFolder("private");
	Path checkpointPrivateDirectory = new Path(checkpointPrivateFolder.getPath());

	File checkpointSharedFolder = temporaryFolder.newFolder("shared");
	Path checkpointSharedDirectory = new Path(checkpointSharedFolder.getPath());

	FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
	int fileStateSizeThreshold = 1024;
	FsCheckpointStreamFactory checkpointStreamFactory =
		new FsCheckpointStreamFactory(fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold);

	String localFolder = "local";
	temporaryFolder.newFolder(localFolder);

	int sstFileCount = 6;
	Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);

	try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
		Map<StateHandleID, StreamStateHandle> sstFiles =
			rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, new CloseableRegistry());

		for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
			assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
		}
	}
}
Example #3
Source File: RocksDBStateUploaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
	File checkpointPrivateFolder = temporaryFolder.newFolder("private");
	Path checkpointPrivateDirectory = new Path(checkpointPrivateFolder.getPath());

	File checkpointSharedFolder = temporaryFolder.newFolder("shared");
	Path checkpointSharedDirectory = new Path(checkpointSharedFolder.getPath());

	FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
	int fileStateSizeThreshold = 1024;
	int writeBufferSize = 4096;
	FsCheckpointStreamFactory checkpointStreamFactory =
		new FsCheckpointStreamFactory(
			fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold, writeBufferSize);

	String localFolder = "local";
	temporaryFolder.newFolder(localFolder);

	int sstFileCount = 6;
	Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);

	try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
		Map<StateHandleID, StreamStateHandle> sstFiles =
			rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, new CloseableRegistry());

		for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
			assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
		}
	}
}
Example #4
Source File: RocksDBStateUploaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
	File checkpointPrivateFolder = temporaryFolder.newFolder("private");
	org.apache.flink.core.fs.Path checkpointPrivateDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointPrivateFolder);

	File checkpointSharedFolder = temporaryFolder.newFolder("shared");
	org.apache.flink.core.fs.Path checkpointSharedDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointSharedFolder);

	FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
	int fileStateSizeThreshold = 1024;
	int writeBufferSize = 4096;
	FsCheckpointStreamFactory checkpointStreamFactory =
		new FsCheckpointStreamFactory(
			fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold, writeBufferSize);

	String localFolder = "local";
	temporaryFolder.newFolder(localFolder);

	int sstFileCount = 6;
	Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);

	try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
		Map<StateHandleID, StreamStateHandle> sstFiles =
			rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, new CloseableRegistry());

		for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
			assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
		}
	}
}
Example #5
Source File: ChannelStateCheckpointWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFileHandleSize() throws Exception {
	int numChannels = 3;
	int numWritesPerChannel = 4;
	int numBytesPerWrite = 5;
	ChannelStateWriteResult result = new ChannelStateWriteResult();
	ChannelStateCheckpointWriter writer = createWriter(
		result,
		new FsCheckpointStreamFactory(
			getSharedInstance(),
			fromLocalFile(temporaryFolder.newFolder("checkpointsDir")),
			fromLocalFile(temporaryFolder.newFolder("sharedStateDir")),
				numBytesPerWrite - 1,
				numBytesPerWrite - 1).createCheckpointStateOutputStream(EXCLUSIVE));

	InputChannelInfo[] channels = IntStream.range(0, numChannels).mapToObj(i -> new InputChannelInfo(0, i)).toArray(InputChannelInfo[]::new);
	for (int call = 0; call < numWritesPerChannel; call++) {
		for (int channel = 0; channel < numChannels; channel++) {
			write(writer, channels[channel], getData(numBytesPerWrite));
		}
	}
	writer.completeInput();
	writer.completeOutput();

	for (InputChannelStateHandle handle : result.inputChannelStateHandles.get()) {
		assertEquals((Integer.BYTES + numBytesPerWrite) * numWritesPerChannel, handle.getStateSize());
	}
}