Java Code Examples for org.apache.flink.core.fs.RecoverableWriter#ResumeRecoverable

@Override
public RecoverableWriter.ResumeRecoverable persist() throws IOException {
	lock();
	try {
		fileStream.flush();
		openNewPartIfNecessary(userDefinedMinPartSize);

		// We do not stop writing to the current file, we merely limit the upload to the
		// first n bytes of the current file

		return upload.snapshotAndGetRecoverable(fileStream);
	}
	finally {
		unlock();
	}
}
 

@Test(expected = IOException.class)
public void testResumeWithWrongOffset() throws Exception {
	// this is a rather unrealistic scenario, but it is to trigger
	// truncation of the file and try to resume with missing data.

	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable1 = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable2 = stream.persist();
	stream.write(testData3.getBytes(StandardCharsets.UTF_8));

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable1);
	recoveredStream.closeForCommit().commit();

	// this should throw an exception
	final RecoverableFsDataOutputStream newRecoveredStream = writer.recover(recoverable2);
	newRecoveredStream.closeForCommit().commit();
}
 

@Test(expected = IOException.class)
public void testResumeAfterCommit() throws Exception {
	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	stream.closeForCommit().commit();

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable);
	recoveredStream.closeForCommit().commit();
}
 

@Test(expected = IOException.class)
public void testResumeWithWrongOffset() throws Exception {
	// this is a rather unrealistic scenario, but it is to trigger
	// truncation of the file and try to resume with missing data.

	final RecoverableWriter writer = getFileSystem().createRecoverableWriter();
	final Path path = new Path(basePathForTest, "part-0");

	final RecoverableFsDataOutputStream stream = writer.open(path);
	stream.write(testData1.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable1 = stream.persist();
	stream.write(testData2.getBytes(StandardCharsets.UTF_8));

	final RecoverableWriter.ResumeRecoverable recoverable2 = stream.persist();
	stream.write(testData3.getBytes(StandardCharsets.UTF_8));

	final RecoverableFsDataOutputStream recoveredStream = writer.recover(recoverable1);
	recoveredStream.closeForCommit().commit();

	// this should throw an exception
	final RecoverableFsDataOutputStream newRecoveredStream = writer.recover(recoverable2);
	newRecoveredStream.closeForCommit().commit();
}
 

@Override
public InProgressFileWriter<IN, BucketID> resumeFrom(
		final BucketID bucketId,
		final RecoverableFsDataOutputStream stream,
		final RecoverableWriter.ResumeRecoverable resumable,
		final long creationTime) {

	Preconditions.checkNotNull(stream);
	Preconditions.checkNotNull(resumable);

	return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime);
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

@Override
public PartFileWriter<IN, BucketID> resumeFrom(
		final BucketID bucketId,
		final RecoverableFsDataOutputStream stream,
		final RecoverableWriter.ResumeRecoverable resumable,
		final long creationTime) throws IOException {

	Preconditions.checkNotNull(stream);
	Preconditions.checkNotNull(resumable);

	final BulkWriter<IN> writer = writerFactory.create(stream);
	return new BulkPartWriter<>(bucketId, stream, writer, creationTime);
}
 

BucketState(
		final BucketID bucketId,
		final Path bucketPath,
		final long inProgressFileCreationTime,
		@Nullable final RecoverableWriter.ResumeRecoverable inProgressResumableFile,
		final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommittablesPerCheckpoint
) {
	this.bucketId = Preconditions.checkNotNull(bucketId);
	this.bucketPath = Preconditions.checkNotNull(bucketPath);
	this.inProgressFileCreationTime = inProgressFileCreationTime;
	this.inProgressResumableFile = inProgressResumableFile;
	this.committableFilesPerCheckpoint = Preconditions.checkNotNull(pendingCommittablesPerCheckpoint);
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

@Override
public RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(RefCountedFSOutputStream incompletePartFile) throws IOException {
	lastPersistedIndex = uploadedContent.size();

	if (incompletePartFile.getPos() >= 0L) {
		byte[] bytes = readFileContents(incompletePartFile);
		uncompleted = Optional.of(bytes);
	}

	return null;
}
 

@Override
RecoverableWriter.ResumeRecoverable persist() {
	throw new UnsupportedOperationException("Bulk Part Writers do not support \"pause and resume\" operations.");
}
 

SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> getResumeSerializer() {
	return resumeSerializer;
}
 

OutputStreamBasedInProgressFileRecoverableSerializer(SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumeSerializer) {
	this.resumeSerializer = resumeSerializer;
}
 

private BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {

		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer = getCommitableSerializer();
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer = getResumableSerializer();

		final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
		final String bucketPathStr = in.readUTF();
		final long creationTime = in.readLong();

		// then get the current resumable stream
		InProgressFileWriter.InProgressFileRecoverable current = null;
		if (in.readBoolean()) {
			current =
				new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(
					SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in));
		}

		final int committableVersion = in.readInt();
		final int numCheckpoints = in.readInt();
		final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints);

		for (int i = 0; i < numCheckpoints; i++) {
			final long checkpointId = in.readLong();
			final int noOfResumables = in.readInt();

			final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(noOfResumables);
			for (int j = 0; j < noOfResumables; j++) {
				final byte[] bytes = new byte[in.readInt()];
				in.readFully(bytes);
				pendingFileRecoverables.add(
					new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes)));
			}
			pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables);
		}

		return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			pendingFileRecoverablePerCheckpoint);
	}
 

@Nullable
RecoverableWriter.ResumeRecoverable getInProgressResumableFile() {
	return inProgressResumableFile;
}
 

public abstract InProgressFileWriter<IN, BucketID> resumeFrom(
final BucketID bucketId,
final RecoverableFsDataOutputStream stream,
final RecoverableWriter.ResumeRecoverable resumable,
final long creationTime) throws IOException;
 

@Override
public RecoverableWriter.ResumeRecoverable persist() throws IOException {
	return null;
}
 

@Override
public RecoverableFsDataOutputStream recover(RecoverableWriter.ResumeRecoverable resumable) throws IOException {
	return null;
}
 

/**
 * Used upon recovery from a failure to recover a {@link PartFileWriter writer}.
 * @param bucketId the id of the bucket this writer is writing to.
 * @param stream the filesystem-specific output stream to use when writing to the filesystem.
 * @param resumable the state of the stream we are resurrecting.
 * @param creationTime the creation time of the stream.
 * @return the recovered {@link PartFileWriter writer}.
 * @throws IOException
 */
PartFileWriter<IN, BucketID> resumeFrom(
	final BucketID bucketId,
	final RecoverableFsDataOutputStream stream,
	final RecoverableWriter.ResumeRecoverable resumable,
	final long creationTime) throws IOException;
 

/**
 * Creates a snapshot of this MultiPartUpload, from which the upload can be resumed.
 *
 * @param incompletePartFile The file containing the in-progress part which has not yet reached the minimum
 *                           part size in order to be uploaded.
 *
 * @return The {@link RecoverableWriter.ResumeRecoverable ResumeRecoverable} which
 * can be used to resume the upload.
 */
RecoverableWriter.ResumeRecoverable snapshotAndGetRecoverable(
		@Nullable final RefCountedFSOutputStream incompletePartFile) throws IOException;