org.apache.flink.runtime.io.network.util.DeserializationUtils Java Examples

The following examples show how to use org.apache.flink.runtime.io.network.util.DeserializationUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void verifyDeserializationResults(
		Queue<BufferConsumer> queue,
		RecordDeserializer<SerializationTestType> deserializer,
		ArrayDeque<SerializationTestType> expectedRecords,
		int numRequiredBuffers,
		int numValues) throws Exception {
	int assertRecords = 0;
	for (int j = 0; j < numRequiredBuffers; j++) {
		Buffer buffer = buildSingleBuffer(queue.remove());
		deserializer.setNextBuffer(buffer);

		assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
	}
	Assert.assertEquals(numValues, assertRecords);
}
Example #2
Source File: SpanningRecordSerializationTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Iterates over the provided records and tests whether {@link SpanningRecordSerializer} and {@link RecordDeserializer}
 * interact as expected.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private static void testSerializationRoundTrip(
		Iterable<SerializationTestType> records,
		int segmentSize,
		RecordSerializer<SerializationTestType> serializer,
		RecordDeserializer<SerializationTestType> deserializer)
	throws Exception {
	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer, segmentSize);

	int numRecords = 0;
	for (SerializationTestType record : records) {

		serializedRecords.add(record);

		numRecords++;

		// serialize record
		serializer.serializeRecord(record);
		if (serializer.copyToBufferBuilder(serializationResult.getBufferBuilder()).isFullBuffer()) {
			// buffer is full => start deserializing
			deserializer.setNextBuffer(serializationResult.buildBuffer());

			numRecords -= DeserializationUtils.deserializeRecords(serializedRecords, deserializer);

			// move buffers as long as necessary (for long records)
			while ((serializationResult = setNextBufferForSerializer(serializer, segmentSize)).isFullBuffer()) {
				deserializer.setNextBuffer(serializationResult.buildBuffer());
			}
		}
	}

	// deserialize left over records
	deserializer.setNextBuffer(serializationResult.buildBuffer());

	while (!serializedRecords.isEmpty()) {
		SerializationTestType expected = serializedRecords.poll();

		SerializationTestType actual = expected.getClass().newInstance();
		RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(actual);

		Assert.assertTrue(result.isFullRecord());
		Assert.assertEquals(expected, actual);
		numRecords--;
	}

	// assert that all records have been serialized and deserialized
	Assert.assertEquals(0, numRecords);
	Assert.assertFalse(serializer.hasSerializedData());
	Assert.assertFalse(deserializer.hasUnfinishedData());
}
Example #3
Source File: RecordWriterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * The results of emitting records via BroadcastPartitioner or broadcasting records directly are the same,
 * that is all the target channels can receive the whole outputs.
 *
 * @param isBroadcastEmit whether using {@link RecordWriter#broadcastEmit(IOReadableWritable)} or not
 */
private void emitRecordWithBroadcastPartitionerOrBroadcastEmitRecord(boolean isBroadcastEmit) throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final ChannelSelector selector = new OutputEmitter(ShipStrategyType.BROADCAST, 0);
	final RecordWriter<SerializationTestType> writer = RecordWriter.createRecordWriter(partitionWriter, selector, 0, "test");
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);

		if (isBroadcastEmit) {
			writer.broadcastEmit(record);
		} else {
			writer.emit(record);
		}
	}

	final int requiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(requiredBuffers, queues[i].size());

		final ArrayDeque<SerializationTestType> expectedRecords = serializedRecords.clone();
		int assertRecords = 0;
		for (int j = 0; j < requiredBuffers; j++) {
			Buffer buffer = buildSingleBuffer(queues[i].remove());
			deserializer.setNextBuffer(buffer);

			assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
		}
		Assert.assertEquals(numValues, assertRecords);
	}
}
Example #4
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Iterates over the provided records and tests whether {@link SpanningRecordSerializer} and {@link RecordDeserializer}
 * interact as expected.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private static void testSerializationRoundTrip(
		Iterable<SerializationTestType> records,
		int segmentSize,
		RecordSerializer<SerializationTestType> serializer,
		RecordDeserializer<SerializationTestType> deserializer)
	throws Exception {
	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer, segmentSize);

	int numRecords = 0;
	for (SerializationTestType record : records) {

		serializedRecords.add(record);

		numRecords++;

		// serialize record
		serializer.serializeRecord(record);
		if (serializer.copyToBufferBuilder(serializationResult.getBufferBuilder()).isFullBuffer()) {
			// buffer is full => start deserializing
			deserializer.setNextBuffer(serializationResult.buildBuffer());

			numRecords -= DeserializationUtils.deserializeRecords(serializedRecords, deserializer);

			// move buffers as long as necessary (for long records)
			while ((serializationResult = setNextBufferForSerializer(serializer, segmentSize)).isFullBuffer()) {
				deserializer.setNextBuffer(serializationResult.buildBuffer());
			}
		}
	}

	// deserialize left over records
	deserializer.setNextBuffer(serializationResult.buildBuffer());

	while (!serializedRecords.isEmpty()) {
		SerializationTestType expected = serializedRecords.poll();

		SerializationTestType actual = expected.getClass().newInstance();
		RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(actual);

		Assert.assertTrue(result.isFullRecord());
		Assert.assertEquals(expected, actual);
		numRecords--;
	}

	// assert that all records have been serialized and deserialized
	Assert.assertEquals(0, numRecords);
	Assert.assertFalse(serializer.hasSerializedData());
	Assert.assertFalse(deserializer.hasUnfinishedData());
}
Example #5
Source File: RecordWriterTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * The results of emitting records via BroadcastPartitioner or broadcasting records directly are the same,
 * that is all the target channels can receive the whole outputs.
 *
 * @param isBroadcastEmit whether using {@link RecordWriter#broadcastEmit(IOReadableWritable)} or not
 */
private void emitRecordWithBroadcastPartitionerOrBroadcastEmitRecord(boolean isBroadcastEmit) throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final ChannelSelector selector = new OutputEmitter(ShipStrategyType.BROADCAST, 0);
	final RecordWriter<SerializationTestType> writer = new RecordWriterBuilder()
		.setChannelSelector(selector)
		.setTimeout(0)
		.build(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);

		if (isBroadcastEmit) {
			writer.broadcastEmit(record);
		} else {
			writer.emit(record);
		}
	}

	final int requiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(requiredBuffers, queues[i].size());

		final ArrayDeque<SerializationTestType> expectedRecords = serializedRecords.clone();
		int assertRecords = 0;
		for (int j = 0; j < requiredBuffers; j++) {
			Buffer buffer = buildSingleBuffer(queues[i].remove());
			deserializer.setNextBuffer(buffer);

			assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
		}
		Assert.assertEquals(numValues, assertRecords);
	}
}
Example #6
Source File: SpanningRecordSerializationTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Iterates over the provided records and tests whether {@link SpanningRecordSerializer} and {@link RecordDeserializer}
 * interact as expected.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private static void testSerializationRoundTrip(
		Iterable<SerializationTestType> records,
		int segmentSize,
		RecordSerializer<SerializationTestType> serializer,
		RecordDeserializer<SerializationTestType> deserializer)
	throws Exception {
	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer, segmentSize);

	int numRecords = 0;
	for (SerializationTestType record : records) {

		serializedRecords.add(record);

		numRecords++;

		// serialize record
		serializer.serializeRecord(record);
		if (serializer.copyToBufferBuilder(serializationResult.getBufferBuilder()).isFullBuffer()) {
			// buffer is full => start deserializing
			deserializer.setNextBuffer(serializationResult.buildBuffer());

			numRecords -= DeserializationUtils.deserializeRecords(serializedRecords, deserializer);

			// move buffers as long as necessary (for long records)
			while ((serializationResult = setNextBufferForSerializer(serializer, segmentSize)).isFullBuffer()) {
				deserializer.setNextBuffer(serializationResult.buildBuffer());
			}
		}
	}

	// deserialize left over records
	deserializer.setNextBuffer(serializationResult.buildBuffer());

	while (!serializedRecords.isEmpty()) {
		SerializationTestType expected = serializedRecords.poll();

		SerializationTestType actual = expected.getClass().newInstance();
		RecordDeserializer.DeserializationResult result = deserializer.getNextRecord(actual);

		Assert.assertTrue(result.isFullRecord());
		Assert.assertEquals(expected, actual);
		numRecords--;
	}

	// assert that all records have been serialized and deserialized
	Assert.assertEquals(0, numRecords);
	Assert.assertFalse(serializer.hasSerializedData());
	Assert.assertFalse(deserializer.hasUnfinishedData());
}