Java Code Examples for org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream#getPos()

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

/**
 * Adds a part to the uploads without any size limitations.
 *
 * <p>This method is non-blocking and does not wait for the part upload to complete.
 *
 * @param file The file with the part data.
 *
 * @throws IOException If this method throws an exception, the RecoverableS3MultiPartUpload
 *                     should not be used any more, but recovered instead.
 */
@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	// this is to guarantee that nobody is
	// writing to the file we are uploading.
	checkState(file.isClosed());

	final CompletableFuture<PartETag> future = new CompletableFuture<>();
	uploadsInProgress.add(future);

	final long partLength = file.getPos();
	currentUploadInfo.registerNewPart(partLength);

	file.retain(); // keep the file while the async upload still runs
	uploadThreadPool.execute(new UploadTask(s3AccessHelper, currentUploadInfo, file, future));
}
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * <p>Data buffered locally which is less than
 * {@link org.apache.flink.fs.s3.common.FlinkS3FileSystem#S3_MULTIPART_MIN_PART_SIZE S3_MULTIPART_MIN_PART_SIZE},
 * and cannot be uploaded as part of the MPU and set to S3 as independent objects.
 *
 * <p>This implementation currently blocks until all part uploads are complete and returns
 * a completed future.
 */
@Override
public S3Recoverable snapshotAndGetRecoverable(@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException {

	final String incompletePartObjectName = safelyUploadSmallPart(incompletePartFile);

	// make sure all other uploads are complete
	// this currently makes the method blocking,
	// to be made non-blocking in the future
	awaitPendingPartsUpload();

	final String objectName = currentUploadInfo.getObjectName();
	final String uploadId = currentUploadInfo.getUploadId();
	final List<PartETag> completedParts = currentUploadInfo.getCopyOfEtagsOfCompleteParts();
	final long sizeInBytes = currentUploadInfo.getExpectedSizeInBytes();

	if (incompletePartObjectName == null) {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes);
	} else {
		return new S3Recoverable(objectName, uploadId, completedParts, sizeInBytes, incompletePartObjectName, incompletePartFile.getPos());
	}
}
 

@Nullable
private String safelyUploadSmallPart(@Nullable RefCountedFSOutputStream file) throws IOException {

	if (file == null || file.getPos() == 0L) {
		return null;
	}

	// first, upload the trailing data file. during that time, other in-progress uploads may complete.
	final String incompletePartObjectName = createIncompletePartObjectName();
	file.retain();
	try {
		s3AccessHelper.putObject(incompletePartObjectName, file.getInputFile());
	}
	finally {
		file.release();
	}
	return incompletePartObjectName;
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

@Override
public void uploadPart(RefCountedFSOutputStream file) throws IOException {
	numParts++;
	numBytes += file.getPos();

	uploadedContent.add(readFileContents(file));
}