org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream Java Examples

The following examples show how to use org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3RecoverableFsDataOutputStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
Example #2
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
Example #3
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
Example #4
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
Example #5
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream newStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(tmpFileCreator, Optional.empty());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			0L);
}
Example #6
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
Example #7
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
Example #8
Source File: S3RecoverableFsDataOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Before
public void beforeTest() throws IOException {
	fileProvider = new TestFileProvider(TEMP_FOLDER);

	multipartUploadUnderTest = new TestMultipartUpload(fileProvider);

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.openNew(fileProvider);

	streamUnderTest = new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			0L
	);
}
Example #9
Source File: S3RecoverableFsDataOutputStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static S3RecoverableFsDataOutputStream recoverStream(
		final RecoverableMultiPartUpload upload,
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final long userDefinedMinPartSize,
		final long bytesBeforeCurrentPart) throws IOException {

	checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE);

	final RefCountedBufferingFileStream fileStream = boundedBufferingFileStream(
			tmpFileCreator,
			upload.getIncompletePart());

	return new S3RecoverableFsDataOutputStream(
			upload,
			tmpFileCreator,
			fileStream,
			userDefinedMinPartSize,
			bytesBeforeCurrentPart);
}
Example #10
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
Example #11
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
Example #12
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
Example #13
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFileWithStream, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}
Example #14
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
Example #15
Source File: S3RecoverableFsDataOutputStream.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
Example #16
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFileWithStream.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}
Example #17
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
Example #18
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFile.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}
Example #19
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
Example #20
Source File: RecoverableMultiPartUploadImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
Example #21
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
Example #22
Source File: S3RecoverableFsDataOutputStreamTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
Example #23
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}
Example #24
Source File: S3RecoverableFsDataOutputStream.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void uploadCurrentAndOpenNewPart(long fileLength) throws IOException {
	bytesBeforeCurrentPart += fileLength;
	closeAndUploadPart();

	// initialize a new temp file
	fileStream = RefCountedBufferingFileStream.openNew(tmpFileProvider);
}
Example #25
Source File: RecoverableMultiPartUploadImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private RefCountedBufferingFileStream writeContent(byte[] content) throws IOException {
	final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID());
	final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW);

	final RefCountedBufferingFileStream testStream =
			new RefCountedBufferingFileStream(RefCountedFile.newFile(newFile, out), BUFFER_SIZE);

	testStream.write(content, 0, content.length);
	return testStream;
}
Example #26
Source File: RecoverableMultiPartUploadImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private S3Recoverable uploadObject(byte[] content) throws IOException {
	final RefCountedBufferingFileStream incompletePartFile = writeContent(content);
	incompletePartFile.flush();

	// as in the production code, we assume that a file containing
	// a in-progress part is flushed but not closed before being passed
	// to the uploader.

	return multiPartUploadUnderTest.snapshotAndGetRecoverable(incompletePartFile);
}
Example #27
Source File: RecoverableMultiPartUploadImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalStateException.class)
public void uploadingNonClosedFileAsCompleteShouldThroughException() throws IOException {
	final byte[] incompletePart = bytesOf("!!!");

	final RefCountedBufferingFileStream incompletePartFile =
			writeContent(incompletePart);

	multiPartUploadUnderTest.uploadPart(incompletePartFile);
}
Example #28
Source File: S3RecoverableFsDataOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private S3RecoverableFsDataOutputStream reopenStreamUnderTestAfterRecovery() throws IOException {
	final long bytesBeforeCurrentPart = multipartUploadUnderTest.numBytes;
	final Optional<File> incompletePart = multipartUploadUnderTest.getIncompletePart();

	RefCountedBufferingFileStream fileStream = RefCountedBufferingFileStream.restore(fileProvider, incompletePart.get());
	multipartUploadUnderTest.discardUnpersistedData();

	return new S3RecoverableFsDataOutputStream(
			multipartUploadUnderTest,
			fileProvider,
			fileStream,
			USER_DEFINED_MIN_PART_SIZE,
			bytesBeforeCurrentPart
	);
}
Example #29
Source File: S3RecoverableFsDataOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void discardingUnpersistedUploadedData() throws IOException {
	streamUnderTest.write(bytesOf("hello"));

	streamUnderTest.persist();
	streamUnderTest.write(randomBuffer(RefCountedBufferingFileStream.BUFFER_SIZE + 1));
	streamUnderTest = reopenStreamUnderTestAfterRecovery();

	streamUnderTest.write(bytesOf(" world"));
	streamUnderTest.closeForCommit().commit();

	assertThat(multipartUploadUnderTest, hasContent(bytesOf("hello world")));
}
Example #30
Source File: S3RecoverableFsDataOutputStream.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private static RefCountedBufferingFileStream boundedBufferingFileStream(
		final FunctionWithException<File, RefCountedFile, IOException> tmpFileCreator,
		final Optional<File> incompletePart) throws IOException {

	if (!incompletePart.isPresent()) {
		return RefCountedBufferingFileStream.openNew(tmpFileCreator);
	}

	final File file = incompletePart.get();
	return RefCountedBufferingFileStream.restore(tmpFileCreator, file);
}