org.apache.flink.testutils.serialization.types.SerializationTestType Java Examples

@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 

/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 

@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 

@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 

/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 

@Test
public void testHandleMixedLargeRecords() throws Exception {
	final int numValues = 99;
	final int segmentSize = 32 * 1024;

	List<SerializationTestType> originalRecords = new ArrayList<>((numValues + 1) / 2);
	LargeObjectType genLarge = new LargeObjectType();
	Random rnd = new Random();

	for (int i = 0; i < numValues; i++) {
		if (i % 2 == 0) {
			originalRecords.add(new IntType(42));
		} else {
			originalRecords.add(genLarge.getRandom(rnd));
		}
	}

	testSerializationRoundTrip(originalRecords, segmentSize);
}
 

/**
 * Iterates over the provided records to deserialize, verifies the results and stats
 * the number of full records.
 *
 * @param records records to be deserialized
 * @param deserializer the record deserializer
 * @return the number of full deserialized records
 */
public static int deserializeRecords(
		ArrayDeque<SerializationTestType> records,
		RecordDeserializer<SerializationTestType> deserializer) throws Exception {
	int deserializedRecords = 0;

	while (!records.isEmpty()) {
		SerializationTestType expected = records.poll();
		SerializationTestType actual = expected.getClass().newInstance();

		if (deserializer.getNextRecord(actual).isFullRecord()) {
			Assert.assertEquals(expected, actual);
			deserializedRecords++;
		} else {
			records.addFirst(expected);
			break;
		}
	}

	return deserializedRecords;
}
 

private static void testSequenceOfTypes(Iterable<SerializationTestType> sequence, int segmentSize) throws Exception {

		List<SerializationTestType> elements = new ArrayList<>(512);
		TestOutputView outView = new TestOutputView(segmentSize);

		// write
		for (SerializationTestType type : sequence) {
			// serialize the record
			type.write(outView);
			elements.add(type);
		}
		outView.close();

		// check the records
		TestInputView inView = new TestInputView(outView.segments);

		for (SerializationTestType reference : elements) {
			SerializationTestType result = reference.getClass().newInstance();
			result.read(inView);
			assertEquals(reference, result);
		}
	}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 

private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 

public void testUnconsumedBuffer(
		RecordSerializer<SerializationTestType> serializer,
		RecordDeserializer<SerializationTestType> deserializer,
		SerializationTestType record,
		int segmentSize,
		byte... leftOverBytes) throws Exception {
	try (ByteArrayOutputStream unconsumedBytes = new ByteArrayOutputStream()) {
		serializer.serializeRecord(record);

		BufferAndSerializerResult serializationResult = setNextBufferForSerializer(serializer, segmentSize);

		if (serializer.copyToBufferBuilder(serializationResult.getBufferBuilder()).isFullBuffer()) {
			// buffer is full => start deserializing
			Buffer buffer = serializationResult.buildBuffer();
			writeBuffer(buffer.readOnlySlice().getNioBufferReadable(), unconsumedBytes);
			deserializer.setNextBuffer(buffer);
			assertUnconsumedBuffer(unconsumedBytes, deserializer.getUnconsumedBuffer());

			deserializer.getNextRecord(record.getClass().newInstance());

			// move buffers as long as necessary (for long records)
			while ((serializationResult = setNextBufferForSerializer(serializer, segmentSize)).isFullBuffer()) {
				buffer = serializationResult.buildBuffer();

				if (serializationResult.isFullRecord()) {
					buffer = appendLeftOverBytes(buffer, leftOverBytes);
				}

				writeBuffer(buffer.readOnlySlice().getNioBufferReadable(), unconsumedBytes);
				deserializer.setNextBuffer(buffer);
				assertUnconsumedBuffer(unconsumedBytes, deserializer.getUnconsumedBuffer());

				deserializer.getNextRecord(record.getClass().newInstance());
			}
		}
	}
}
 

private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 

@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 

/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 

@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 

private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

/**
 * FLINK-17780: Tests that a shared buffer(or memory segment) of a buffer builder is only freed when all consumers
 * are closed.
 */
@Test
public void testRandomEmitAndBufferRecycling() throws Exception {
	int recordSize = 8;

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(2, 2 * recordSize);
	final KeepingPartitionWriter partitionWriter = new KeepingPartitionWriter(bufferProvider) {
		@Override
		public int getNumberOfSubpartitions() {
			return 2;
		}
	};
	final BroadcastRecordWriter<SerializationTestType> writer = new BroadcastRecordWriter<>(partitionWriter, 0, "test");

	// force materialization of both buffers for easier availability tests
	List<Buffer> buffers = Arrays.asList(bufferProvider.requestBuffer(), bufferProvider.requestBuffer());
	buffers.forEach(Buffer::recycleBuffer);
	assertEquals(2, bufferProvider.getNumberOfAvailableBuffers());

	// fill first buffer
	writer.randomEmit(new IntType(1), 0);
	writer.broadcastEmit(new IntType(2));
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());

	// simulate consumption of first buffer consumer; this should not free buffers
	assertEquals(1, partitionWriter.getAddedBufferConsumers(0).size());
	closeConsumer(partitionWriter, 0, 2 * recordSize);
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());

	// use second buffer
	writer.broadcastEmit(new IntType(3));
	assertEquals(0, bufferProvider.getNumberOfAvailableBuffers());

	// fully free first buffer
	assertEquals(2, partitionWriter.getAddedBufferConsumers(1).size());
	closeConsumer(partitionWriter, 1, recordSize);
	assertEquals(1, bufferProvider.getNumberOfAvailableBuffers());
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

private static BufferAndSerializerResult setNextBufferForSerializer(
		RecordSerializer<SerializationTestType> serializer,
		int segmentSize) throws IOException {
	// create a bufferBuilder with some random starting offset to properly test handling buffer slices in the
	// deserialization code.
	int startingOffset = segmentSize > 2 ? RANDOM.nextInt(segmentSize / 2) : 0;
	BufferBuilder bufferBuilder = createFilledBufferBuilder(segmentSize + startingOffset, startingOffset);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	bufferConsumer.build().recycleBuffer();

	return new BufferAndSerializerResult(
		bufferBuilder,
		bufferConsumer,
		serializer.copyToBufferBuilder(bufferBuilder));
}
 

/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 

private void testSerializationRoundTrip(Iterable<SerializationTestType> records, int segmentSize) throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{ tempFolder.getRoot().getAbsolutePath() });

	testSerializationRoundTrip(records, segmentSize, serializer, deserializer);
}
 

protected void verifyDeserializationResults(
		Queue<BufferConsumer> queue,
		RecordDeserializer<SerializationTestType> deserializer,
		ArrayDeque<SerializationTestType> expectedRecords,
		int numRequiredBuffers,
		int numValues) throws Exception {
	int assertRecords = 0;
	for (int j = 0; j < numRequiredBuffers; j++) {
		Buffer buffer = buildSingleBuffer(queue.remove());
		deserializer.setNextBuffer(buffer);

		assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
	}
	Assert.assertEquals(numValues, assertRecords);
}
 

@Test
public void testEmptyRecords() throws IOException {
	final int segmentSize = 11;

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final BufferBuilder bufferBuilder1 = createBufferBuilder(segmentSize);

	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD,
		serializer.copyToBufferBuilder(bufferBuilder1));

	SerializationTestType emptyRecord = new SerializationTestType() {
		@Override
		public SerializationTestType getRandom(Random rnd) {
			throw new UnsupportedOperationException();
		}

		@Override
		public int length() {
			throw new UnsupportedOperationException();
		}

		@Override
		public void write(DataOutputView out) {}

		@Override
		public void read(DataInputView in) {}

		@Override
		public int hashCode() {
			throw new UnsupportedOperationException();
		}

		@Override
		public boolean equals(Object obj) {
			throw new UnsupportedOperationException();
		}
	};

	serializer.serializeRecord(emptyRecord);
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.copyToBufferBuilder(bufferBuilder1));

	serializer.reset();
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.copyToBufferBuilder(bufferBuilder1));

	serializer.reset();
	Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL,
		serializer.copyToBufferBuilder(bufferBuilder1));

	final BufferBuilder bufferBuilder2 = createBufferBuilder(segmentSize);
	Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD,
		serializer.copyToBufferBuilder(bufferBuilder2));
}