org.apache.flink.runtime.io.disk.RandomAccessOutputView Java Examples

private void testSerialize(
	BinaryRow row, MemorySegment[] memorySegments,
	Map<MemorySegment, Integer> msIndex, BinaryRowSerializer serializer, int position,
	int rowSize) throws IOException {
	RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
	out.skipBytesToWrite(position);
	row.setTotalSize(rowSize);

	// this `row` contains random bytes, and now we're going to serialize `rowSize` bytes
	// (not including the row header) of the contents
	serializer.serializeToPages(row, out);

	// let's see how many segments we have written
	int segNumber = msIndex.get(out.getCurrentSegment()) + 1;
	int lastSegSize = out.getCurrentPositionInSegment();

	// now deserialize from the written segments
	ArrayList<MemorySegment> segments = new ArrayList<>(Arrays.asList(memorySegments).subList(0, segNumber));
	RandomAccessInputView input = new RandomAccessInputView(segments, 64, lastSegSize);
	input.skipBytesToRead(position);
	BinaryRow mapRow = serializer.mapFromPages(input);

	assertEquals(row, mapRow);
}
 

@Test
public void testSerStringToKryo() throws IOException {
	KryoSerializer<BinaryString> serializer = new KryoSerializer<>(
		BinaryString.class, new ExecutionConfig());

	BinaryString string = BinaryString.fromString("hahahahaha");
	RandomAccessOutputView out = new RandomAccessOutputView(
		new MemorySegment[]{MemorySegmentFactory.wrap(new byte[1024])}, 64);
	serializer.serialize(string, out);

	RandomAccessInputView input = new RandomAccessInputView(
		new ArrayList<>(Collections.singletonList(out.getCurrentSegment())), 64, 64);
	BinaryString newStr = serializer.deserialize(input);

	assertEquals(string, newStr);
}
 

private void testSerialize(
	BinaryRowData row, MemorySegment[] memorySegments,
	Map<MemorySegment, Integer> msIndex, BinaryRowDataSerializer serializer, int position,
	int rowSize) throws IOException {
	RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
	out.skipBytesToWrite(position);
	row.setTotalSize(rowSize);

	// this `row` contains random bytes, and now we're going to serialize `rowSize` bytes
	// (not including the row header) of the contents
	serializer.serializeToPages(row, out);

	// let's see how many segments we have written
	int segNumber = msIndex.get(out.getCurrentSegment()) + 1;
	int lastSegSize = out.getCurrentPositionInSegment();

	// now deserialize from the written segments
	ArrayList<MemorySegment> segments = new ArrayList<>(Arrays.asList(memorySegments).subList(0, segNumber));
	RandomAccessInputView input = new RandomAccessInputView(segments, 64, lastSegSize);
	input.skipBytesToRead(position);
	BinaryRowData mapRow = serializer.mapFromPages(input);

	assertEquals(row, mapRow);
}
 

@Test
public void testSerStringToKryo() throws IOException {
	KryoSerializer<BinaryStringData> serializer = new KryoSerializer<>(
		BinaryStringData.class, new ExecutionConfig());

	BinaryStringData string = BinaryStringData.fromString("hahahahaha");
	RandomAccessOutputView out = new RandomAccessOutputView(
		new MemorySegment[]{MemorySegmentFactory.wrap(new byte[1024])}, 64);
	serializer.serialize(string, out);

	RandomAccessInputView input = new RandomAccessInputView(
		new ArrayList<>(Collections.singletonList(out.getCurrentSegment())), 64, 64);
	StringData newStr = serializer.deserialize(input);

	assertEquals(string, newStr);
}
 

@Test
public void testGenericObject() throws Exception {

	GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
	TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());

	BinaryRow row = new BinaryRow(4);
	BinaryRowWriter writer = new BinaryRowWriter(row);
	writer.writeInt(0, 0);

	BinaryGeneric<MyObj> myObj1 = new BinaryGeneric<>(new MyObj(0, 1), genericSerializer);
	writer.writeGeneric(1, myObj1);
	BinaryGeneric<MyObj> myObj2 = new BinaryGeneric<>(new MyObj(123, 5.0), genericSerializer);
	myObj2.ensureMaterialized();
	writer.writeGeneric(2, myObj2);
	BinaryGeneric<MyObj> myObj3 = new BinaryGeneric<>(new MyObj(1, 1), genericSerializer);
	writer.writeGeneric(3, myObj3);
	writer.complete();

	assertTestGenericObjectRow(row, genericSerializer);

	// getBytes from var-length memorySegments.
	BinaryRowSerializer serializer = new BinaryRowSerializer(4);
	MemorySegment[] memorySegments = new MemorySegment[3];
	ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
	for (int i = 0; i < 3; i++) {
		memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
		memorySegmentList.add(memorySegments[i]);
	}
	RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
	serializer.serializeToPages(row, out);

	BinaryRow mapRow = serializer.mapFromPages(new RandomAccessInputView(memorySegmentList, 64));
	assertTestGenericObjectRow(mapRow, genericSerializer);
}
 

@Test
public void testHashAndCopy() throws IOException {
	MemorySegment[] segments = new MemorySegment[3];
	for (int i = 0; i < 3; i++) {
		segments[i] = MemorySegmentFactory.wrap(new byte[64]);
	}
	RandomAccessOutputView out = new RandomAccessOutputView(segments, 64);
	BinaryRowSerializer serializer = new BinaryRowSerializer(2);

	BinaryRow row = new BinaryRow(2);
	BinaryRowWriter writer = new BinaryRowWriter(row);
	writer.writeString(0, BinaryString.fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahah"));
	writer.writeString(1, BinaryString.fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahaa"));
	writer.complete();
	serializer.serializeToPages(row, out);

	ArrayList<MemorySegment> segmentList = new ArrayList<>(Arrays.asList(segments));
	RandomAccessInputView input = new RandomAccessInputView(segmentList, 64, 64);

	BinaryRow mapRow = serializer.mapFromPages(input);
	assertEquals(row, mapRow);
	assertEquals(row.getString(0), mapRow.getString(0));
	assertEquals(row.getString(1), mapRow.getString(1));
	assertNotEquals(row.getString(0), mapRow.getString(1));

	// test if the hash code before and after serialization are the same
	assertEquals(row.hashCode(), mapRow.hashCode());
	assertEquals(row.getString(0).hashCode(), mapRow.getString(0).hashCode());
	assertEquals(row.getString(1).hashCode(), mapRow.getString(1).hashCode());

	// test if the copy method produce a row with the same contents
	assertEquals(row.copy(), mapRow.copy());
	assertEquals(row.getString(0).copy(), mapRow.getString(0).copy());
	assertEquals(row.getString(1).copy(), mapRow.getString(1).copy());
}
 

@Test
public void testGenericObject() throws Exception {

	GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
	TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());
	RawValueDataSerializer<MyObj> binarySerializer = new RawValueDataSerializer<>(genericSerializer);

	BinaryRowData row = new BinaryRowData(4);
	BinaryRowWriter writer = new BinaryRowWriter(row);
	writer.writeInt(0, 0);

	RawValueData<MyObj> myObj1 = RawValueData.fromObject(new MyObj(0, 1));
	writer.writeRawValue(1, myObj1, binarySerializer);
	RawValueData<MyObj> myObj2 = RawValueData.fromObject(new MyObj(123, 5.0));
	writer.writeRawValue(2, myObj2, binarySerializer);
	RawValueData<MyObj> myObj3 = RawValueData.fromObject(new MyObj(1, 1));
	writer.writeRawValue(3, myObj3, binarySerializer);
	writer.complete();

	assertTestGenericObjectRow(row, genericSerializer);

	// getBytes from var-length memorySegments.
	BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(4);
	MemorySegment[] memorySegments = new MemorySegment[3];
	ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
	for (int i = 0; i < 3; i++) {
		memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
		memorySegmentList.add(memorySegments[i]);
	}
	RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
	serializer.serializeToPages(row, out);

	BinaryRowData mapRow = serializer.mapFromPages(new RandomAccessInputView(memorySegmentList, 64));
	assertTestGenericObjectRow(mapRow, genericSerializer);
}
 

@Test
public void testHashAndCopy() throws IOException {
	MemorySegment[] segments = new MemorySegment[3];
	for (int i = 0; i < 3; i++) {
		segments[i] = MemorySegmentFactory.wrap(new byte[64]);
	}
	RandomAccessOutputView out = new RandomAccessOutputView(segments, 64);
	BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(2);

	BinaryRowData row = new BinaryRowData(2);
	BinaryRowWriter writer = new BinaryRowWriter(row);
	writer.writeString(0, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahah"));
	writer.writeString(1, fromString("hahahahahahahahahahahahahahahahahahahhahahahahahahahahaa"));
	writer.complete();
	serializer.serializeToPages(row, out);

	ArrayList<MemorySegment> segmentList = new ArrayList<>(Arrays.asList(segments));
	RandomAccessInputView input = new RandomAccessInputView(segmentList, 64, 64);

	BinaryRowData mapRow = serializer.mapFromPages(input);
	assertEquals(row, mapRow);
	assertEquals(row.getString(0), mapRow.getString(0));
	assertEquals(row.getString(1), mapRow.getString(1));
	assertNotEquals(row.getString(0), mapRow.getString(1));

	// test if the hash code before and after serialization are the same
	assertEquals(row.hashCode(), mapRow.hashCode());
	assertEquals(row.getString(0).hashCode(), mapRow.getString(0).hashCode());
	assertEquals(row.getString(1).hashCode(), mapRow.getString(1).hashCode());

	// test if the copy method produce a row with the same contents
	assertEquals(row.copy(), mapRow.copy());
	assertEquals(
		((BinaryStringData) row.getString(0)).copy(),
		((BinaryStringData) mapRow.getString(0)).copy());
	assertEquals(
		((BinaryStringData) row.getString(1)).copy(),
		((BinaryStringData) mapRow.getString(1)).copy());
}
 

final SeekableDataOutputView getWriteView() {
	if (this.overwriteBuffer == null) {
		this.overwriteBuffer = new RandomAccessOutputView(this.partitionBuffers, this.memorySegmentSize);
	}
	return this.overwriteBuffer;
}
 

final SeekableDataOutputView getWriteView() {
	if (this.overwriteBuffer == null) {
		this.overwriteBuffer = new RandomAccessOutputView(this.partitionBuffers, this.memorySegmentSize);
	}
	return this.overwriteBuffer;
}
 

final SeekableDataOutputView getWriteView() {
	if (this.overwriteBuffer == null) {
		this.overwriteBuffer = new RandomAccessOutputView(this.partitionBuffers, this.memorySegmentSize);
	}
	return this.overwriteBuffer;
}