Java Code Examples for org.apache.flink.core.io.SimpleVersionedSerialization#writeVersionAndSerialize()

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

private StateInitializationContext getStateContext() throws Exception {
	// Create a mock split.
	byte[] serializedSplitWithVersion = SimpleVersionedSerialization
			.writeVersionAndSerialize(new MockSourceSplitSerializer(), MOCK_SPLIT);

	// Crate the state context.
	OperatorStateStore operatorStateStore = createOperatorStateStore();
	StateInitializationContext stateContext = new StateInitializationContextImpl(
			false,
			operatorStateStore,
			null,
			null,
			null);

	// Update the context.
	stateContext.getOperatorStateStore()
				.getListState(SourceOperator.SPLITS_STATE_DESC)
				.update(Collections.singletonList(serializedSplitWithVersion));

	return stateContext;
}
 

@Test
@Ignore
public void prepareDeserializationEmpty() throws IOException {

	final String scenarioName = "empty";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);

	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(),
		bucketState);
	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}
 

@Test
@Ignore
public void prepareDeserializationOnlyInProgress() throws IOException {

	final String scenarioName = "only-in-progress";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(), bucketState);

	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

private void serializeV2(BucketState<BucketID> state, DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), dataOutputView);
	dataOutputView.writeUTF(state.getBucketPath().toString());
	dataOutputView.writeLong(state.getInProgressFileCreationTime());

	// put the current open part file
	if (state.hasInProgressFileRecoverable()) {
		final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable();
		dataOutputView.writeBoolean(true);
		SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView);
	} else {
		dataOutputView.writeBoolean(false);
	}

	// put the map of pending files per checkpoint
	final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = state.getPendingFileRecoverablesPerCheckpoint();

	dataOutputView.writeInt(pendingFileRecoverableSerializer.getVersion());

	dataOutputView.writeInt(pendingFileRecoverables.size());

	for (Entry<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFilesForCheckpoint : pendingFileRecoverables.entrySet()) {
		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverableList = pendingFilesForCheckpoint.getValue();

		dataOutputView.writeLong(pendingFilesForCheckpoint.getKey());
		dataOutputView.writeInt(pendingFileRecoverableList.size());

		for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : pendingFileRecoverableList) {
			byte[] serialized = pendingFileRecoverableSerializer.serialize(pendingFileRecoverable);
			dataOutputView.writeInt(serialized.length);
			dataOutputView.write(serialized);
		}
	}
}
 

private static void prepareDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException {

		final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
		FileUtils.deleteDirectory(Paths.get(scenarioPath.toString() + "-template").toFile());
		Files.createDirectories(scenarioPath);

		final int noOfPendingCheckpoints = 5;

		final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);

		final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

		final Bucket<String, String> bucket = createNewBucket(testBucketPath);

		BucketState<String> bucketState = null;
		// pending for checkpoints
		for (int i = 0; i < noOfPendingCheckpoints; i++) {
			// write 10 bytes to the in progress file
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			// every checkpoint would produce a pending file
			bucketState = bucket.onReceptionOfCheckpoint(i);
		}

		if (withInProgress) {
			// create a in progress file
			bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

			// 5 pending files and 1 in progress file
			bucketState = bucket.onReceptionOfCheckpoint(noOfPendingCheckpoints);
		}

		final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(bucketStateSerializer(), bucketState);

		Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);

		// copy the scenario file to a template directory.
		// it is because that the test `testSerializationFull` would change the in progress file to pending files.
		moveToTemplateDirectory(scenarioPath);
	}
 

private byte[] serialize(T value) throws IOException {
	return SimpleVersionedSerialization.writeVersionAndSerialize(serializer, value);
}
 

private void serializeV1(final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(resumeSerializer, outputStreamBasedInProgressRecoverable.getResumeRecoverable(), dataOutputView);
}
 

private void serializeV1(final OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(commitSerializer, outputStreamBasedPendingFileRecoverable.getCommitRecoverable(), dataOutputView);
}