Java Code Examples for org.apache.flink.core.fs.Path#fromLocalFile()

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

@Test
public void testSavepointsInOneDirectoryDefaultLocation() throws Exception {
	final Path defaultSavepointDir = Path.fromLocalFile(tmp.newFolder());

	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()), defaultSavepointDir, new JobID(), FILE_SIZE_THRESHOLD);

	final FsCheckpointStorageLocation savepointLocation = (FsCheckpointStorageLocation)
			storage.initializeLocationForSavepoint(52452L, null);

	// all state types should be in the expected location
	assertParent(defaultSavepointDir, savepointLocation.getCheckpointDirectory());
	assertParent(defaultSavepointDir, savepointLocation.getSharedStateDirectory());
	assertParent(defaultSavepointDir, savepointLocation.getTaskOwnedStateDirectory());

	// cleanup
	savepointLocation.disposeOnFailure();
}
 

@Test
public void testWriteCompressedFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<String> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(String.class)
	);

	stream.map(str -> str).addSink(
			StreamingFileSink.forBulkFormat(
					testPath,
					CompressWriters.forExtractor(new DefaultExtractor<String>()).withHadoopCompression(TEST_CODEC_NAME)
			).build());

	env.execute();

	validateResults(folder, testData, new CompressionCodecFactory(configuration).getCodecByName(TEST_CODEC_NAME));
}
 

@Override
public Path call() throws IOException {
	final File file = blobService.getFile(jobID, blobKey);

	if (isDirectory) {
		Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target);
		return directory;
	} else {
		//noinspection ResultOfMethodCallIgnored
		file.setExecutable(isExecutable);
		return Path.fromLocalFile(file);
	}

}
 

/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 

@Override
public Path call() throws IOException {
	final File file = blobService.getFile(jobID, blobKey);

	if (isDirectory) {
		Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target);
		return directory;
	} else {
		//noinspection ResultOfMethodCallIgnored
		file.setExecutable(isExecutable);
		return Path.fromLocalFile(file);
	}

}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertNull(handle);
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testDisposeDeletesFile() throws Exception {
	File file = tempFolder.newFile();
	writeTestData(file);
	assertTrue(file.exists());

	FileStateHandle handle = new FileStateHandle(Path.fromLocalFile(file), file.length());
	handle.discardState();
	assertFalse(file.exists());
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Test
public void testWriteSequenceFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Tuple2<Long, String>> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {

			})
	);

	stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() {
		@Override
		public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception {
			return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1));
		}
	}).addSink(
		StreamingFileSink.forBulkFormat(
			testPath,
			new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2")
		).build());

	env.execute();

	validateResults(folder, testData);
}
 

@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 

@Test
public void testWriteSequenceFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Tuple2<Long, String>> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {

			})
	);

	stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() {
		@Override
		public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception {
			return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1));
		}
	}).addSink(
		StreamingFileSink.forBulkFormat(
			testPath,
			new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2")
		).build());

	env.execute();

	validateResults(folder, testData);
}
 

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 

public Path randomTempPath() throws IOException {
	return Path.fromLocalFile(tmp.newFolder());
}
 

@Test(expected = IllegalArgumentException.class)
public void testWrongParameters() throws Exception {
	// this should fail
	new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000, relativePaths);
}
 

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 

@Before
public void createStateDirectories() throws IOException {
	exclusiveStateDir = Path.fromLocalFile(TMP.newFolder("exclusive"));
	sharedStateDir = Path.fromLocalFile(TMP.newFolder("shared"));
}