org.apache.flink.testutils.serialization.types.SerializationTestTypeFactory Java Examples

@Test
public void testLargeSpanningRecordUnconsumedBufferWithLeftOverBytes() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});

	deserializer.clear();

	testUnconsumedBuffer(
		serializer,
		deserializer,
		Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY),
		1,
		new byte[] {42, 43, 44});
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

@Test
public void testHasSerializedData() throws IOException {
	final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	final SerializationTestType randomIntRecord = Util.randomRecord(SerializationTestTypeFactory.INT);

	Assert.assertFalse(serializer.hasSerializedData());

	serializer.serializeRecord(randomIntRecord);
	Assert.assertTrue(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder1 = createBufferBuilder(16);
	serializer.copyToBufferBuilder(bufferBuilder1);
	Assert.assertFalse(serializer.hasSerializedData());

	final BufferBuilder bufferBuilder2 = createBufferBuilder(8);
	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	Assert.assertFalse(serializer.hasSerializedData());

	serializer.reset();
	serializer.copyToBufferBuilder(bufferBuilder2);
	// Buffer builder full!
	Assert.assertTrue(serializer.hasSerializedData());
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testLargeSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.BYTE_ARRAY), 1);
}
 

/**
 * Test both for spanning records and for handling the length buffer, that's why it's going byte by byte.
 */
@Test
public void testSpanningRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1);
}
 

@Test
public void testSmallRecordUnconsumedBuffer() throws Exception {
	RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<>();
	RecordDeserializer<SerializationTestType> deserializer =
		new SpillingAdaptiveSpanningRecordDeserializer<>(
			new String[]{tempFolder.getRoot().getAbsolutePath()});

	testUnconsumedBuffer(serializer, deserializer, Util.randomRecord(SerializationTestTypeFactory.INT), 1024);
}
 

@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

/**
 * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}.
 */
@Test
public void testBroadcastEmitRecord() throws Exception {
	final int numberOfChannels = 4;
	final int bufferSize = 32;
	final int numValues = 8;
	final int serializationLength = 4;

	@SuppressWarnings("unchecked")
	final Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);
	final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	final RecordWriter<SerializationTestType> writer = createRecordWriter(partitionWriter);
	final RecordDeserializer<SerializationTestType> deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(
		new String[]{ tempFolder.getRoot().getAbsolutePath() });

	final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<>();
	final Iterable<SerializationTestType> records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT);
	for (SerializationTestType record : records) {
		serializedRecords.add(record);
		writer.broadcastEmit(record);
	}

	final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength));
	if (isBroadcastWriter) {
		assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers());
	} else {
		assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers());
	}

	for (int i = 0; i < numberOfChannels; i++) {
		assertEquals(numRequiredBuffers, queues[i].size());
		verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues);
	}
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testSequenceOfIntegersWithUnalignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2047);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testSequenceOfIntegersWithAlignedBuffers() {
	try {
		final int numInts = 1000000;

		testSequenceOfTypes(Util.randomRecords(numInts, SerializationTestTypeFactory.INT), 2048);

	} catch (Exception e) {
		e.printStackTrace();
		fail("Test encountered an unexpected exception.");
	}
}
 

@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedBuffers () throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedBuffers () throws Exception {
	final int segmentSize = 31;
	final int numValues = 248;

	testSerializationRoundTrip(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithUnalignedSegments() throws Exception {
	final int segmentSize = 31;
	final int numValues = 248; // least common multiple => last record should align

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsWithAlignedSegments() throws Exception {
	final int segmentSize = 64;
	final int numValues = 64;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}
 

@Test
public void testIntRecordsSpanningMultipleSegments() throws Exception {
	final int segmentSize = 1;
	final int numValues = 10;

	test(Util.randomRecords(numValues, SerializationTestTypeFactory.INT), segmentSize);
}