org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream Java Examples

UploadTask(
		final S3AccessHelper s3AccessHelper,
		final MultiPartUploadInfo currentUpload,
		final RefCountedFSOutputStream file,
		final CompletableFuture<PartETag> future) {

	checkNotNull(currentUpload);

	this.objectName = currentUpload.getObjectName();
	this.uploadId = currentUpload.getUploadId();
	this.partNumber = currentUpload.getNumberOfRegisteredParts();

	// these are limits put by Amazon
	checkArgument(partNumber >= 1  && partNumber <= 10_000);

	this.s3AccessHelper = checkNotNull(s3AccessHelper);
	this.file = checkNotNull(file);
	this.future = checkNotNull(future);
}
 

UploadTask(
		final S3AccessHelper s3AccessHelper,
		final MultiPartUploadInfo currentUpload,
		final RefCountedFSOutputStream file,
		final CompletableFuture<PartETag> future) {

	checkNotNull(currentUpload);

	this.objectName = currentUpload.getObjectName();
	this.uploadId = currentUpload.getUploadId();
	this.partNumber = currentUpload.getNumberOfRegisteredParts();

	// these are limits put by Amazon
	checkArgument(partNumber >= 1  && partNumber <= 10_000);

	this.s3AccessHelper = checkNotNull(s3AccessHelper);
	this.file = checkNotNull(file);
	this.future = checkNotNull(future);
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Single constructor to initialize all. Actual setup of the parts happens in the
 * factory methods.
 */
S3RecoverableFsDataOutputStream(
		RecoverableMultiPartUpload upload,
		FunctionWithException<File, RefCountedFile, IOException> tempFileCreator,
		RefCountedFSOutputStream initialTmpFile,
		long userDefinedMinPartSize,
		long bytesBeforeCurrentPart) {

	checkArgument(bytesBeforeCurrentPart >= 0L);

	this.upload = checkNotNull(upload);
	this.tmpFileProvider = checkNotNull(tempFileCreator);
	this.userDefinedMinPartSize = userDefinedMinPartSize;

	this.fileStream = initialTmpFile;
	this.bytesBeforeCurrentPart = bytesBeforeCurrentPart;
}
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

/**
 * Single constructor to initialize all. Actual setup of the parts happens in the
 * factory methods.
 */
S3RecoverableFsDataOutputStream(
		RecoverableMultiPartUpload upload,
		FunctionWithException<File, RefCountedFileWithStream, IOException> tempFileCreator,
		RefCountedFSOutputStream initialTmpFile,
		long userDefinedMinPartSize,
		long bytesBeforeCurrentPart) {

	checkArgument(bytesBeforeCurrentPart >= 0L);

	this.upload = checkNotNull(upload);
	this.tmpFileProvider = checkNotNull(tempFileCreator);
	this.userDefinedMinPartSize = userDefinedMinPartSize;

	this.fileStream = initialTmpFile;
	this.bytesBeforeCurrentPart = bytesBeforeCurrentPart;
}
 

UploadTask(
		final S3AccessHelper s3AccessHelper,
		final MultiPartUploadInfo currentUpload,
		final RefCountedFSOutputStream file,
		final CompletableFuture<PartETag> future) {

	checkNotNull(currentUpload);

	this.objectName = currentUpload.getObjectName();
	this.uploadId = currentUpload.getUploadId();
	this.partNumber = currentUpload.getNumberOfRegisteredParts();

	// these are limits put by Amazon
	checkArgument(partNumber >= 1  && partNumber <= 10_000);

	this.s3AccessHelper = checkNotNull(s3AccessHelper);
	this.file = checkNotNull(file);
	this.future = checkNotNull(future);
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Single constructor to initialize all. Actual setup of the parts happens in the
 * factory methods.
 */
S3RecoverableFsDataOutputStream(
		RecoverableMultiPartUpload upload,
		FunctionWithException<File, RefCountedFile, IOException> tempFileCreator,
		RefCountedFSOutputStream initialTmpFile,
		long userDefinedMinPartSize,
		long bytesBeforeCurrentPart) {

	checkArgument(bytesBeforeCurrentPart >= 0L);

	this.upload = checkNotNull(upload);
	this.tmpFileProvider = checkNotNull(tempFileCreator);
	this.userDefinedMinPartSize = userDefinedMinPartSize;

	this.fileStream = initialTmpFile;
	this.bytesBeforeCurrentPart = bytesBeforeCurrentPart;
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 

private static byte[] readFileContents(RefCountedFSOutputStream file) throws IOException {
	final byte[] content = new byte[MathUtils.checkedDownCast(file.getPos())];
	File inputFile = file.getInputFile();
	long bytesRead = new FileInputStream(inputFile).read(content, 0,  MathUtils.checkedDownCast(inputFile.length()));
	Assert.assertEquals(file.getPos(), bytesRead);
	return content;
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

private static byte[] readFileContents(RefCountedFSOutputStream file) throws IOException {
	final byte[] content = new byte[MathUtils.checkedDownCast(file.getPos())];
	File inputFile = file.getInputFile();
	long bytesRead = new FileInputStream(inputFile).read(content, 0,  MathUtils.checkedDownCast(inputFile.length()));
	Assert.assertEquals(file.getPos(), bytesRead);
	return content;
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

private static byte[] readFileContents(RefCountedFSOutputStream file) throws IOException {
	final byte[] content = new byte[MathUtils.checkedDownCast(file.getPos())];
	File inputFile = file.getInputFile();
	long bytesRead = new FileInputStream(inputFile).read(content, 0,  MathUtils.checkedDownCast(inputFile.length()));
	Assert.assertEquals(file.getPos(), bytesRead);
	return content;
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * @param incompletePartFile The file containing the in-progress part which has not yet reached the minimum
 *                           part size in order to be uploaded.
 *
 * @return The {@link RecoverableWriter.ResumeRecoverable ResumeRecoverable} which
 * can be used to resume the upload.
 */
RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(
		@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException;
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
void uploadPart(final RefCountedFSOutputStream file) throws IOException;
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
void uploadPart(final RefCountedFSOutputStream file) throws IOException;
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * @param incompletePartFile The file containing the in-progress part which has not yet reached the minimum
 *                           part size in order to be uploaded.
 *
 * @return The {@link RecoverableWriter.ResumeRecoverable ResumeRecoverable} which
 * can be used to resume the upload.
 */
RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(
		@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException;
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
void uploadPart(final RefCountedFSOutputStream file) throws IOException;
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * @param incompletePartFile The file containing the in-progress part which has not yet reached the minimum
 *                           part size in order to be uploaded.
 *
 * @return The {@link RecoverableWriter.ResumeRecoverable ResumeRecoverable} which
 * can be used to resume the upload.
 */
RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(
		@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException;