org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView Java Examples

static void deserializeFromPages(BinaryRowData reuse, ChannelReaderInputView inView,
		BinaryRowDataSerializer buildSideSerializer) throws IOException {
	if (shouldAdvance(
			inView.getCurrentSegmentLimit() - inView.getCurrentPositionInSegment(),
			buildSideSerializer)) {
		inView.advance();
	}
	MemorySegment segment = (reuse.getSegments() != null) ? reuse.getSegments()[0] : null;

	int length = toLength(inView.getCurrentSegment().getLong(inView.getCurrentPositionInSegment()));
	inView.skipBytesToRead(8);

	if (segment == null || segment.size() < length) {
		segment = MemorySegmentFactory.wrap(new byte[length]);
	}
	inView.readFully(segment.getHeapMemory(), 0, length);
	reuse.pointTo(segment, 0, length);
}
 

private void newSpilledIterator(int channelID, long offset) throws IOException {
	ChannelWithMeta channel = spilledChannelIDs.get(channelID);
	currentChannelID = channelID;

	// close current reader first.
	closeCurrentFileReader();

	// calculate segment number
	int segmentNum = (int) (offset / segmentSize);
	long seekPosition = segmentNum * segmentSize;

	// new reader.
	this.fileReader = ioManager.createBlockChannelReader(channel.getChannel());
	if (offset > 0) {
		// seek to the beginning of that segment
		fileReader.seekToPosition(seekPosition);
	}
	ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(
		fileReader, getReadMemory(), channel.getBlockCount() - segmentNum,
		channel.getNumBytesInLastBlock(), false, offset - seekPosition
	);
	this.currentIterator = new BinaryRowChannelInputViewIterator(inView, binaryRowSerializer);
}
 

private void newSpilledIterator(int channelID, long offset) throws IOException {
	ChannelWithMeta channel = spilledChannelIDs.get(channelID);
	currentChannelID = channelID;

	// close current reader first.
	closeCurrentFileReader();

	// calculate segment number
	int segmentNum = (int) (offset / segmentSize);
	long seekPosition = segmentNum * segmentSize;

	// new reader.
	this.fileReader = ioManager.createBlockChannelReader(channel.getChannel());
	if (offset > 0) {
		// seek to the beginning of that segment
		fileReader.seekToPosition(seekPosition);
	}
	ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(
		fileReader, getReadMemory(), channel.getBlockCount() - segmentNum,
		channel.getNumBytesInLastBlock(), false, offset - seekPosition
	);
	this.currentIterator = new BinaryRowChannelInputViewIterator(inView, binaryRowSerializer);
}
 

static void deserializeFromPages(BinaryRow reuse, ChannelReaderInputView inView,
		BinaryRowSerializer buildSideSerializer) throws IOException {
	if (shouldAdvance(
			inView.getCurrentSegmentLimit() - inView.getCurrentPositionInSegment(),
			buildSideSerializer)) {
		inView.advance();
	}
	MemorySegment segment = (reuse.getSegments() != null) ? reuse.getSegments()[0] : null;

	int length = toLength(inView.getCurrentSegment().getLong(inView.getCurrentPositionInSegment()));
	inView.skipBytesToRead(8);

	if (segment == null || segment.size() < length) {
		segment = MemorySegmentFactory.wrap(new byte[length]);
	}
	inView.readFully(segment.getHeapMemory(), 0, length);
	reuse.pointTo(segment, 0, length);
}
 

public ChannelReaderInputViewIterator(BlockChannelReader<MemorySegment> reader, LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this.accessors = accessors;
	this.freeMemTarget = freeMemTarget;
	this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
}
 

public BinaryRowChannelInputViewIterator(
		ChannelReaderInputView inView,
		List<MemorySegment> freeMemTarget,
		BinaryRowDataSerializer serializer) {
	this.inView = inView;
	this.freeMemTarget = freeMemTarget;
	this.serializer = serializer;
}
 

public ChannelReaderInputViewIterator(BlockChannelReader<MemorySegment> reader, LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this.accessors = accessors;
	this.freeMemTarget = freeMemTarget;
	this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
}
 

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 

public BinaryRowChannelInputViewIterator(
		ChannelReaderInputView inView,
		List<MemorySegment> freeMemTarget,
		BinaryRowSerializer serializer) {
	this.inView = inView;
	this.freeMemTarget = freeMemTarget;
	this.serializer = serializer;
}
 

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 

public ChannelReaderInputViewIterator(BlockChannelReader<MemorySegment> reader, LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this.accessors = accessors;
	this.freeMemTarget = freeMemTarget;
	this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
}
 

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 

@Test
public void testFlushFullMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 0, NUM_RECORDS);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =0;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testFlushPartialMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 1, NUM_RECORDS - 1);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =1;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testWriteReadSmallRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteAndReadLongRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		final int k1 = rec.f0;
		final String v1 = rec.f1;
		final int k2 = readRec.f0;
		final String v2 = readRec.f1;
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testReadTooMany() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());

	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();

	// read and re-generate all records and compare them
	try {
		final Tuple2<Integer, String> readRec = new Tuple2<>();
		for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
			generator.next(rec);
			serializer.deserialize(readRec, inView);
			final int k1 = rec.f0;
			final String v1 = rec.f1;
			final int k2 = readRec.f0;
			final String v2 = readRec.f1;
			Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
		}
		Assert.fail("Expected an EOFException which did not occur.");
	}
	catch (EOFException eofex) {
		// expected
	}
	catch (Throwable t) {
		// unexpected
		Assert.fail("Unexpected Exception: " + t.getMessage());
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testReadWithoutKnownBlockCount() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteReadOneBufferOnly() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, 1);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, 1);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

public BinaryRowChannelInputViewIterator(
		ChannelReaderInputView inView,
		BinaryRowDataSerializer serializer) {
	this(inView, new ArrayList<>(), serializer);
}
 

@Test
public void testWriteReadNotAll() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteReadNotAll() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT / 2; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteReadOneBufferOnly() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, 1);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, 1);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testReadWithoutKnownBlockCount() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testReadTooMany() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());

	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();

	// read and re-generate all records and compare them
	try {
		final Tuple2<Integer, String> readRec = new Tuple2<>();
		for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
			generator.next(rec);
			serializer.deserialize(readRec, inView);
			final int k1 = rec.f0;
			final String v1 = rec.f1;
			final int k2 = readRec.f0;
			final String v2 = readRec.f1;
			Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
		}
		Assert.fail("Expected an EOFException which did not occur.");
	}
	catch (EOFException eofex) {
		// expected
	}
	catch (Throwable t) {
		// unexpected
		Assert.fail("Unexpected Exception: " + t.getMessage());
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteAndReadLongRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		final int k1 = rec.f0;
		final String v1 = rec.f1;
		final int k2 = readRec.f0;
		final String v2 = readRec.f1;
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteReadSmallRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testFlushPartialMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 1, NUM_RECORDS - 1);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =1;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testFlushFullMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 0, NUM_RECORDS);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =0;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

public BinaryRowChannelInputViewIterator(
		ChannelReaderInputView inView,
		BinaryRowSerializer serializer) {
	this(inView, new ArrayList<>(), serializer);
}