Java Code Examples for org.apache.flink.core.fs.RecoverableFsDataOutputStream#write()

@Test(expected = IOException.class)
public void testResumeWithWrongOffset() throws Exception {
	// this is a rather unrealistic scenario, but it is to trigger
	// truncation of the file and try to resume with missing data.

	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable1 = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable2 = stream.persist();
	stream.write(testData3.getBytes(StandardCharsets.UTF_8));

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable1);
	recoveredStream.closeForCommit().commit();

	// this should throw an exception
	final RecoverableFsDataOutputStream newRecoveredStream = writer.recover(recoverable2);
	newRecoveredStream.closeForCommit().commit();
}
 

@Test
public void testCallingDeleteObjectTwiceDoesNotThroughException() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	S3Recoverable recoverable = (S3Recoverable) stream.persist();

	stream.closeForCommit().commit();

	// still the data is there as we have not deleted them from the tmp object
	final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
	Assert.assertEquals(testData1, content);

	boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable);
	Assert.assertTrue(successfullyDeletedState);

	boolean unsuccessfulDeletion = writer.cleanupRecoverableState(recoverable);
	Assert.assertFalse(unsuccessfulDeletion);
}
 

@Test(expected = IOException.class)
public void testResumeAfterCommit() throws Exception {
	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().commit();

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable);
	recoveredStream.closeForCommit().commit();
}
 

@Test(expected = IOException.class)
public void testResumeAfterCommit() throws Exception {
	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().commit();

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable);
	recoveredStream.closeForCommit().commit();
}
 

@Test(expected = IOException.class)
public void testResumeWithWrongOffset() throws Exception {
	// this is a rather unrealistic scenario, but it is to trigger
	// truncation of the file and try to resume with missing data.

	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable1 = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable2 = stream.persist();
	stream.write(testData3.getBytes(StandardCharsets.UTF_8));

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable1);
	recoveredStream.closeForCommit().commit();

	// this should throw an exception
	final RecoverableFsDataOutputStream newRecoveredStream = writer.recover(recoverable2);
	newRecoveredStream.closeForCommit().commit();
}
 

@Test(expected = FileNotFoundException.class)
public void testCleanupRecoverableState() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	S3Recoverable recoverable = (S3Recoverable) stream.persist();

	stream.closeForCommit().commit();

	// still the data is there as we have not deleted them from the tmp object
	final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
	Assert.assertEquals(testData1, content);

	boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable);
	Assert.assertTrue(successfullyDeletedState);

	// this should throw the exception as we deleted the file.
	getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
}
 

@Test
public void testCallingDeleteObjectTwiceDoesNotThroughException() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	S3Recoverable recoverable = (S3Recoverable) stream.persist();

	stream.closeForCommit().commit();

	// still the data is there as we have not deleted them from the tmp object
	final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
	Assert.assertEquals(testData1, content);

	boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable);
	Assert.assertTrue(successfullyDeletedState);

	boolean unsuccessfulDeletion = writer.cleanupRecoverableState(recoverable);
	Assert.assertFalse(unsuccessfulDeletion);
}
 

@Test(expected = IOException.class)
public void testResumeAfterCommit() throws Exception {
	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().commit();

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable);
	recoveredStream.closeForCommit().commit();
}
 

@Test(expected = FileNotFoundException.class)
public void testCleanupRecoverableState() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	S3Recoverable recoverable = (S3Recoverable) stream.persist();

	stream.closeForCommit().commit();

	// still the data is there as we have not deleted them from the tmp object
	final String content = getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
	Assert.assertEquals(testData1, content);

	boolean successfullyDeletedState = writer.cleanupRecoverableState(recoverable);
	Assert.assertTrue(successfullyDeletedState);

	// this should throw the exception as we deleted the file.
	getContentsOfFile(new Path('/' + recoverable.incompleteObjectName()));
}
 

@Test
public void testCommitAfterNormalClose() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	stream.closeForCommit().commit();

	Assert.assertEquals(testData1, getContentsOfFile(path));
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

@Test(expected = IOException.class)
public void testExceptionWritingAfterCloseForCommit() throws Exception {
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = getFileSystem().createRecoverableWriter().open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().getRecoverable();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));
}
 

private void testResumeAfterMultiplePersist(
		final String persistName,
		final String expectedFinalContents,
		final String firstItemToWrite,
		final String secondItemToWrite,
		final String thirdItemToWrite) throws Exception {

	final Path path = new Path(basePathForTest, "part-0");
	final RecoverableWriter initWriter = getRecoverableWriter();

	final Map<String, RecoverableWriter.ResumeRecoverable> recoverables = new HashMap<>(4);
	try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) {
		recoverables.put(INIT_EMPTY_PERSIST, stream.persist());

		stream.write(bytesOf(firstItemToWrite));

		recoverables.put(INTERM_WITH_STATE_PERSIST, stream.persist());
		recoverables.put(INTERM_WITH_NO_ADDITIONAL_STATE_PERSIST, stream.persist());

		// and write some more data
		stream.write(bytesOf(secondItemToWrite));

		recoverables.put(FINAL_WITH_EXTRA_STATE, stream.persist());
	}

	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> serializer = initWriter.getResumeRecoverableSerializer();
	final byte[] serializedRecoverable = serializer.serialize(recoverables.get(persistName));

	// get a new serializer from a new writer to make sure that no pre-initialized state leaks in.
	final RecoverableWriter newWriter = getRecoverableWriter();
	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer();
	final RecoverableWriter.ResumeRecoverable recoveredRecoverable =
			deserializer.deserialize(serializer.getVersion(), serializedRecoverable);

	final RecoverableFsDataOutputStream recoveredStream = newWriter.recover(recoveredRecoverable);
	recoveredStream.write(bytesOf(thirdItemToWrite));
	recoveredStream.closeForCommit().commit();

	Assert.assertEquals(expectedFinalContents, getContentsOfFile(path));
}
 

@Test
public void testCommitAfterNormalClose() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	stream.closeForCommit().commit();

	Assert.assertEquals(testData1, getContentsOfFile(path));
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

@Test(expected = IOException.class)
public void testExceptionWritingAfterCloseForCommit() throws Exception {
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = getFileSystem().createRecoverableWriter().open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().getRecoverable();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));
}
 

private void testResumeAfterMultiplePersist(
		final String persistName,
		final String expectedFinalContents,
		final String firstItemToWrite,
		final String secondItemToWrite,
		final String thirdItemToWrite) throws Exception {

	final Path path = new Path(basePathForTest, "part-0");
	final RecoverableWriter initWriter = getRecoverableWriter();

	final Map<String, RecoverableWriter.ResumeRecoverable> recoverables = new HashMap<>(4);
	try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) {
		recoverables.put(INIT_EMPTY_PERSIST, stream.persist());

		stream.write(bytesOf(firstItemToWrite));

		recoverables.put(INTERM_WITH_STATE_PERSIST, stream.persist());
		recoverables.put(INTERM_WITH_NO_ADDITIONAL_STATE_PERSIST, stream.persist());

		// and write some more data
		stream.write(bytesOf(secondItemToWrite));

		recoverables.put(FINAL_WITH_EXTRA_STATE, stream.persist());
	}

	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> serializer = initWriter.getResumeRecoverableSerializer();
	final byte[] serializedRecoverable = serializer.serialize(recoverables.get(persistName));

	// get a new serializer from a new writer to make sure that no pre-initialized state leaks in.
	final RecoverableWriter newWriter = getRecoverableWriter();
	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer();
	final RecoverableWriter.ResumeRecoverable recoveredRecoverable =
			deserializer.deserialize(serializer.getVersion(), serializedRecoverable);

	final RecoverableFsDataOutputStream recoveredStream = newWriter.recover(recoveredRecoverable);
	recoveredStream.write(bytesOf(thirdItemToWrite));
	recoveredStream.closeForCommit().commit();

	Assert.assertEquals(expectedFinalContents, getContentsOfFile(path));
}
 

@Test
public void testCommitAfterPersist() throws Exception {
	final RecoverableWriter writer = getRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(bytesOf(testData1));
	stream.persist();

	stream.write(bytesOf(testData2));
	stream.closeForCommit().commit();

	Assert.assertEquals(testData1 + testData2, getContentsOfFile(path));
}
 

private RecoverableFsDataOutputStream getOpenStreamToFileWithContent(
		final RecoverableWriter writerUnderTest,
		final Path path,
		final String expectedContent) throws IOException {

	final byte[] content = expectedContent.getBytes(UTF_8);

	final RecoverableFsDataOutputStream streamUnderTest = writerUnderTest.open(path);
	streamUnderTest.write(content);
	return streamUnderTest;
}
 

@Test
public void testCommitAfterRecovery() throws Exception {
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableWriter initWriter = getRecoverableWriter();

	final RecoverableFsDataOutputStream stream = initWriter.open(path);
	stream.write(bytesOf(testData1));

	stream.persist();
	stream.persist();

	// and write some more data
	stream.write(bytesOf(testData2));

	final RecoverableWriter.CommitRecoverable recoverable = stream.closeForCommit().getRecoverable();

	final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable);

	// get a new serializer from a new writer to make sure that no pre-initialized state leaks in.
	final RecoverableWriter newWriter = getRecoverableWriter();

	final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer();
	final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable);

	final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable);
	committer.commitAfterRecovery();

	Assert.assertEquals(testData1 + testData2, getContentsOfFile(path));
}