Java Code Examples for org.apache.flink.core.io.SimpleVersionedSerialization#readVersionAndDeSerialize()

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

@Override
public T next() {
	final byte[] bytes = rawIterator.next();
	try {
		return SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);
	}
	catch (IOException e) {
		throw new FlinkRuntimeException("Failed to deserialize value", e);
	}
}
 

private BucketState<BucketID> deserializeV2(DataInputView dataInputView) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, dataInputView);
	final String bucketPathStr = dataInputView.readUTF();
	final long creationTime = dataInputView.readLong();

	// then get the current resumable stream
	InProgressFileWriter.InProgressFileRecoverable current = null;
	if (dataInputView.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(inProgressFileRecoverableSerializer, dataInputView);
	}

	final int pendingFileRecoverableSerializerVersion = dataInputView.readInt();
	final int numCheckpoints = dataInputView.readInt();
	final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = dataInputView.readLong();
		final int numOfPendingFileRecoverables = dataInputView.readInt();

		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(numOfPendingFileRecoverables);
		for (int j = 0; j < numOfPendingFileRecoverables; j++) {
			final byte[] bytes = new byte[dataInputView.readInt()];
			dataInputView.readFully(bytes);
			pendingFileRecoverables.add(pendingFileRecoverableSerializer.deserialize(pendingFileRecoverableSerializerVersion, bytes));
		}
		pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverables);
	}

	return new BucketState<>(bucketId, new Path(bucketPathStr), creationTime, current, pendingFileRecoverablesPerCheckpoint);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

private BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {

		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer = getCommitableSerializer();
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer = getResumableSerializer();

		final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
		final String bucketPathStr = in.readUTF();
		final long creationTime = in.readLong();

		// then get the current resumable stream
		InProgressFileWriter.InProgressFileRecoverable current = null;
		if (in.readBoolean()) {
			current =
				new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(
					SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in));
		}

		final int committableVersion = in.readInt();
		final int numCheckpoints = in.readInt();
		final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints);

		for (int i = 0; i < numCheckpoints; i++) {
			final long checkpointId = in.readLong();
			final int noOfResumables = in.readInt();

			final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(noOfResumables);
			for (int j = 0; j < noOfResumables; j++) {
				final byte[] bytes = new byte[in.readInt()];
				in.readFully(bytes);
				pendingFileRecoverables.add(
					new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes)));
			}
			pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables);
		}

		return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			pendingFileRecoverablePerCheckpoint);
	}
 

private OutputStreamBasedInProgressFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedInProgressFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(resumeSerializer, dataInputView));
}
 

private OutputStreamBasedPendingFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedPendingFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(commitSerializer, dataInputView));
}
 

private static BucketState<String> readBucketState(final String scenarioName, final int version) throws IOException {
	byte[] bytes = Files.readAllBytes(getSnapshotPath(scenarioName, version));
	return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes);
}