org.apache.flink.core.fs.FSDataOutputStream Java Examples

@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 

/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = baseFolder();
	final String fileName = "myFileName";
	final Path filePath = new Path(folder, fileName);

	final FileStateHandle handle;
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		handle = closeAndGetResult(stream);
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(filePath, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(filePath).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 

@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 

@Override
public BulkWriter<RowData> create(FSDataOutputStream out) throws IOException {
	BulkWriter<RowData> writer = factory.create(out);
	return new BulkWriter<RowData>() {

		@Override
		public void addElement(RowData element) throws IOException {
			writer.addElement(computer.projectColumnsToWrite(element));
		}

		@Override
		public void flush() throws IOException {
			writer.flush();
		}

		@Override
		public void finish() throws IOException {
			writer.finish();
		}
	};
}
 

public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 

@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 

public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "test_name";
	final Path filePath = new Path(folder, fileName);

	final FileSystem fs = spy(new TestFs((path) -> new FailingCloseStream(new File(path.getPath()))));

	FSDataOutputStream stream = createTestStream(fs, folder, fileName);
	stream.write(new byte[] {1, 2, 3, 4, 5});

	try {
		closeAndGetResult(stream);
		fail("Expected IOException");
	}
	catch (IOException ignored) {
		// expected exception
	}

	verify(fs).delete(filePath, false);
}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 

@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
	checkNotNull(filePath, "filePath");

	if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) {
		throw new FileAlreadyExistsException("File already exists: " + filePath);
	}

	final Path parent = filePath.getParent();
	if (parent != null && !mkdirs(parent)) {
		throw new IOException("Mkdirs failed to create " + parent);
	}

	final File file = pathToFile(filePath);
	return new LocalDataOutputStream(file);
}
 

/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = baseFolder();
	final String fileName = "myFileName";
	final Path filePath = new Path(folder, fileName);

	final FileStateHandle handle;
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		handle = closeAndGetResult(stream);
	}

	// must have created a handle
	assertNotNull(handle);
	assertEquals(filePath, handle.getFilePath());

	// the pointer path should exist as a directory
	assertTrue(fs.exists(handle.getFilePath()));
	assertFalse(fs.getFileStatus(filePath).isDir());

	// the contents should be empty
	try (FSDataInputStream in = handle.openInputStream()) {
		assertEquals(-1, in.read());
	}
}
 

@Override
public BulkWriter<RowData> create(FSDataOutputStream out) throws IOException {
	BulkWriter<GenericRecord> writer = factory.create(out);
	AvroRowDataSerializationSchema.SerializationRuntimeConverter converter =
			AvroRowDataSerializationSchema.createRowConverter(rowType);
	Schema schema = AvroSchemaConverter.convertToSchema(rowType);
	return new BulkWriter<RowData>() {

		@Override
		public void addElement(RowData element) throws IOException {
			GenericRecord record = (GenericRecord) converter.convert(schema, element);
			writer.addElement(record);
		}

		@Override
		public void flush() throws IOException {
			writer.flush();
		}

		@Override
		public void finish() throws IOException {
			writer.finish();
		}
	};
}
 

@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 

public long[] write(FSDataOutputStream out) throws IOException {

		long[] partitionOffsets = new long[internalList.size()];

		DataOutputView dov = new DataOutputViewStreamWrapper(out);

		for (int i = 0; i < internalList.size(); ++i) {
			S element = internalList.get(i);
			partitionOffsets[i] = out.getPos();
			getStateMetaInfo().getPartitionStateSerializer().serialize(element, dov);
		}

		return partitionOffsets;
	}
 

@Test
public void testSimpleFileWriteAndRead() throws Exception {
	final Configuration conf = createConfiguration();

	final String testLine = "Hello Upload!";

	FileSystem.initialize(conf);

	final Path path = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR + "/test.txt");
	final FileSystem fs = path.getFileSystem();

	try {
		try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE);
			OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
			writer.write(testLine);
		}

		try (FSDataInputStream in = fs.open(path);
			InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
			BufferedReader reader = new BufferedReader(ir)) {
			String line = reader.readLine();
			assertEquals(testLine, line);
		}
	}
	finally {
		fs.delete(path, false);
	}
}
 

@Override
public long write(FSDataOutputStream out) throws IOException {
	long partitionOffset = out.getPos();

	DataOutputView dov = new DataOutputViewStreamWrapper(out);
	dov.writeInt(backingMap.size());
	for (Map.Entry<K, V> entry: backingMap.entrySet()) {
		getStateMetaInfo().getKeySerializer().serialize(entry.getKey(), dov);
		getStateMetaInfo().getValueSerializer().serialize(entry.getValue(), dov);
	}

	return partitionOffset;
}
 

/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()'
 * call is in progress.
 * <p>
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "this-is-ignored-anyways.file";

	final FileSystem fileSystem = spy(new TestFs((path) -> new BlockerStream()));

	final FSDataOutputStream checkpointStream =
		createTestStream(fileSystem, folder, fileName);

	final OneShotLatch sync = new OneShotLatch();

	final CheckedThread thread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			sync.trigger();
			// that call should now block, because it accesses the position
			closeAndGetResult(checkpointStream);
		}
	};
	thread.start();

	sync.await();
	checkpointStream.close();

	// the thread may or may not fail, that depends on the thread race
	// it is not important for this test, important is that the thread does not freeze/lock up
	try {
		thread.sync();
	} catch (IOException ignored) {}
}
 

private void createFile(Path path, boolean overwrite) throws IOException {
	final WriteMode writeMode = overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE;

	try (FSDataOutputStream out = fileSystem.create(path, writeMode)) {
		out.write(42);
	}
}
 

@Override
public void close() throws IOException {
	final FSDataOutputStream s = this.stream;
	if (s != null) {
		this.stream = null;
		s.close();
	}
}
 

private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException {
	try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) {
		IOUtils.copyBytes(fsInput, lfsOutput);
		//noinspection ResultOfMethodCallIgnored
		new File(targetPath.toString()).setExecutable(executable);
	}
}
 

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 

/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()'
 * call is in progress.
 * <p>
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "this-is-ignored-anyways.file";

	final FileSystem fileSystem = spy(new TestFs((path) -> new BlockerStream()));

	final FSDataOutputStream checkpointStream =
		createTestStream(fileSystem, folder, fileName);

	final OneShotLatch sync = new OneShotLatch();

	final CheckedThread thread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			sync.trigger();
			// that call should now block, because it accesses the position
			closeAndGetResult(checkpointStream);
		}
	};
	thread.start();

	sync.await();
	checkpointStream.close();

	// the thread may or may not fail, that depends on the thread race
	// it is not important for this test, important is that the thread does not freeze/lock up
	try {
		thread.sync();
	} catch (IOException ignored) {}
}
 

@Override
public long write(FSDataOutputStream out) throws IOException {
	long partitionOffset = out.getPos();

	DataOutputView dov = new DataOutputViewStreamWrapper(out);
	dov.writeInt(backingMap.size());
	for (Map.Entry<K, V> entry: backingMap.entrySet()) {
		getStateMetaInfo().getKeySerializer().serialize(entry.getKey(), dov);
		getStateMetaInfo().getValueSerializer().serialize(entry.getValue(), dov);
	}

	return partitionOffset;
}
 

/**
 * Closes the stream successfully and returns a FileStateHandle to the result.
 */
private FileStateHandle closeAndGetResult(FSDataOutputStream stream) throws IOException {
	switch (stateOutputStreamType) {
		case FileBasedState:
			return ((FileBasedStateOutputStream) stream).closeAndGetHandle();
		case FsCheckpointMetaData:
			return ((FsCheckpointMetadataOutputStream) stream).closeAndFinalizeCheckpoint().getMetadataHandle();
		default:
			throw new IllegalStateException("Unsupported checkpoint stream output type.");
	}
}
 

void spillToDiskAndClear(FSDataOutputStream raw) throws IOException {
	if (!isSuppressed) {
		for (ByteBuffer buffer: output) {
			raw.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
				buffer.remaining());
		}
		output.clear();
	}
	isSuppressed = false;
}
 

public static Path expandDirectory(Path file, Path targetDirectory) throws IOException {
	FileSystem sourceFs = file.getFileSystem();
	FileSystem targetFs = targetDirectory.getFileSystem();
	Path rootDir = null;
	try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) {
		ZipEntry entry;
		while ((entry = zis.getNextEntry()) != null) {
			Path relativePath = new Path(entry.getName());
			if (rootDir == null) {
				// the first entry contains the name of the original directory that was zipped
				rootDir = relativePath;
			}

			Path newFile = new Path(targetDirectory, relativePath);
			if (entry.isDirectory()) {
				targetFs.mkdirs(newFile);
			} else {
				try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) {
					// do not close the streams here as it prevents access to further zip entries
					IOUtils.copyBytes(zis, fileStream, false);
				}
			}
			zis.closeEntry();
		}
	}
	return new Path(targetDirectory, rootDir);
}