org.apache.flink.testutils.serialization.types.Util Java Examples

@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 

/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 

@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 

@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 

@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 

@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testRandomTypes() {
	try {
		final int numTypes = 100000;

		// test with an odd buffer size to force many unaligned cases
		testSequenceOfTypes(Util.randomRecords(numTypes), 57);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

@Test
public void testRandomRecords() throws Exception {
	final int segmentSize = 127;
	final int numValues = 100000;

	test(Util.randomRecords(numValues), segmentSize);
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

/**
 * Iterates over the provided records and tests whether the {@link SpanningRecordSerializer} returns the expected
 * {@link RecordSerializer.SerializationResult} values.
 *
 * <p>Only a single {@link MemorySegment} will be allocated.
 *
 * @param records records to test
 * @param segmentSize size for the {@link MemorySegment}
 */
private void test(Util.MockRecords records, int segmentSize) throws Exception {
	final int serializationOverhead = 4; // length encoding

	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();

	// -------------------------------------------------------------------------------------------------------------

	BufferBuilder bufferBuilder = createBufferBuilder(segmentSize);
	int numBytes = 0;
	for (SerializationTestType record : records) {
		serializer.serializeRecord(record);
		RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder);
		numBytes += record.length() + serializationOverhead;

		if (numBytes < segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
		} else if (numBytes == segmentSize) {
			Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD_MEMORY_SEGMENT_FULL, result);
			bufferBuilder = createBufferBuilder(segmentSize);
			numBytes = 0;
		} else {
			Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);

			while (result.isFullBuffer()) {
				numBytes -= segmentSize;
				bufferBuilder = createBufferBuilder(segmentSize);
				result = serializer.copyToBufferBuilder(bufferBuilder);
			}

			Assert.assertTrue(result.isFullRecord());
		}
	}
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testRandomRecords () throws Exception {
	final int segmentSize = 127;
	final int numValues = 10000;

	testSerializationRoundTrip(Util.randomRecords(numValues), segmentSize);
}
 

@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}