org.apache.flink.runtime.io.disk.iomanager.FileIOChannel Java Examples

public CompressedHeaderlessChannelReaderInputView(
		FileIOChannel.ID id,
		IOManager ioManager,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int numBlocks) throws IOException {
	super(0);
	this.numBlocksRemaining = numBlocks;
	this.reader = ioManager.createBufferFileReader(id, this);
	uncompressedBuffer = MemorySegmentFactory.wrap(new byte[compressionBlockSize]);
	decompressor = compressionCodecFactory.getDecompressor();
	cause = new AtomicReference<>();

	BlockCompressor compressor = compressionCodecFactory.getCompressor();
	for (int i = 0; i < 2; i++) {
		MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(
				compressionBlockSize)]);
		reader.readInto(new NetworkBuffer(segment, this));
	}
}
 

public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 

public static BlockChannelWriter<MemorySegment> createBlockChannelWriter(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelWriter(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelWriter(channel, bufferReturnQueue);
	}
}
 

public MutableObjectIterator<Tuple2<BinaryRowData, BinaryRowData>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRowData, BinaryRowData>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 

public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

public MutableObjectIterator<Tuple2<BinaryRow, BinaryRow>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRow, BinaryRow>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 *
 * @param channelIDs    The channels that are to be merged and returned.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
public BinaryMergeIterator<Entry> getMergingIterator(
		List<ChannelWithMeta> channelIDs,
		List<FileIOChannel> openChannels)
		throws IOException {
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}

	final List<MutableObjectIterator<Entry>> iterators = new ArrayList<>(channelIDs.size() + 1);

	for (ChannelWithMeta channel : channelIDs) {
		AbstractChannelReaderInputView view = FileChannelUtil.createInputView(
				ioManager, channel, openChannels, compressionEnable, compressionCodecFactory,
				compressionBlockSize, pageSize);
		iterators.add(channelReaderInputViewIterator(view));
	}

	return new BinaryMergeIterator<>(
			iterators, mergeReusedEntries(channelIDs.size()), mergeComparator());
}
 

int spillPartition(IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			longTable.compressionEnable(),
			longTable.compressionCodecFactory(),
			longTable.compressionBlockSize(),
			segmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 

/**
 * After build phase.
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				compressionEnable,
				compressionCodecFactory,
				compressionBlockSize,
				memorySegmentSize);
		return 1;
	} else {
		return 0;
	}
}
 

public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 

public static BlockChannelReader<MemorySegment> createBlockChannelReader(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelReader(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelReader(channel, bufferReturnQueue);
	}
}
 

public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 

@Test
public void testCompressedView() throws IOException {
	for (int testTime = 0; testTime < 10; testTime++) {
		int testRounds = new Random().nextInt(20000);
		FileIOChannel.ID channel = ioManager.createChannel();
		BufferFileWriter writer = this.ioManager.createBufferFileWriter(channel);
		CompressedHeaderlessChannelWriterOutputView outputView =
				new CompressedHeaderlessChannelWriterOutputView(
						writer,
						compressionFactory,
						BUFFER_SIZE
				);

		for (int i = 0; i < testRounds; i++) {
			outputView.writeInt(i);
		}
		outputView.close();
		int blockCount = outputView.getBlockCount();

		CompressedHeaderlessChannelReaderInputView inputView =
				new CompressedHeaderlessChannelReaderInputView(
						channel,
						ioManager,
						compressionFactory,
						BUFFER_SIZE,
						blockCount
				);

		for (int i = 0; i < testRounds; i++) {
			assertEquals(i, inputView.readInt());
		}
		inputView.close();
	}
}
 

@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID) throws IOException {
	blockLatch.countDown();
	try {
		doneLatch.await();
	} catch (InterruptedException e) {
		throw new IOException("Blocking operation was interrupted.", e);
	}

	return super.createBufferFileWriter(channelID);
}
 

@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID)
		throws IOException {
	BufferFileWriter bufferFileWriter = super.createBufferFileWriter(channelID);
	bufferFileWriter.close();
	return bufferFileWriter;
}
 

private void spill() throws IOException {
	FileIOChannel.ID channel = ioManager.createChannel();

	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers();
	ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments();
	try {
		// spill in memory buffer in zero-copy.
		for (int i = 0; i < numRecordBuffers; i++) {
			writer.writeBlock(segments.get(i));
		}
		LOG.info("here spill the reset buffer data with {} bytes", writer.getSize());
		writer.close();
	} catch (IOException e) {
		writer.closeAndDelete();
		throw e;
	}

	spillSize += numRecordBuffers * segmentSize;
	spilledChannelIDs.add(new ChannelWithMeta(
		channel,
		inMemoryBuffer.getNumRecordBuffers(),
		inMemoryBuffer.getNumBytesInLastBuffer()));
	this.spilledChannelRowOffsets.add(numRows);

	inMemoryBuffer.reset();
}
 

protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
 

public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

/**
 * Open File channels.
 */
public synchronized void addOpenChannels(List<FileIOChannel> toOpen) {
	checkArgument(!closed);
	for (FileIOChannel channel : toOpen) {
		openChannels.add(channel);
		channels.remove(channel.getChannelID());
	}
}
 

/**
 * Merges the sorted runs described by the given Channel IDs into a single sorted run.
 *
 * @param channelIDs   The IDs of the runs' channels.
 * @return The ID and number of blocks of the channel that describes the merged run.
 */
private ChannelWithMeta mergeChannels(List<ChannelWithMeta> channelIDs) throws IOException {
	// the list with the target iterators
	List<FileIOChannel> openChannels = new ArrayList<>(channelIDs.size());
	final BinaryMergeIterator<Entry> mergeIterator =
			getMergingIterator(channelIDs, openChannels);

	// create a new channel writer
	final FileIOChannel.ID mergedChannelID = ioManager.createChannel();
	channelManager.addChannel(mergedChannelID);
	AbstractChannelWriterOutputView output = null;

	int numBytesInLastBlock;
	int numBlocksWritten;
	try {
		output = FileChannelUtil.createOutputView(
				ioManager, mergedChannelID, compressionEnable,
				compressionCodecFactory, compressionBlockSize, pageSize);
		writeMergingOutput(mergeIterator, output);
		numBytesInLastBlock = output.close();
		numBlocksWritten = output.getBlockCount();
	} catch (IOException e) {
		if (output != null) {
			output.close();
			output.getChannel().deleteChannel();
		}
		throw e;
	}

	// remove, close and delete channels
	for (FileIOChannel channel : openChannels) {
		channelManager.removeChannel(channel.getChannelID());
		try {
			channel.closeAndDelete();
		} catch (Throwable ignored) {
		}
	}

	return new ChannelWithMeta(mergedChannelID, numBlocksWritten, numBytesInLastBlock);
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are
 * added to
 * it. The spilling process must free at least one buffer, either in the partition's record
 * buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 *
 * @param ioAccess      The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
int spillPartition(
		IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			compressionEnable,
			compressionCodecFactory,
			compressionBlockSize,
			memorySegmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 

public void sortAndSpill(
		ArrayList<MemorySegment> recordBufferSegments,
		long numElements,
		MemorySegmentPool pool) throws IOException {

	// 1. sort buffer
	BinaryKVInMemorySortBuffer buffer =
			BinaryKVInMemorySortBuffer.createBuffer(
					nKeyComputer, keySerializer, valueSerializer, comparator,
					recordBufferSegments, numElements, pool);
	this.sorter.sort(buffer);

	// 2. spill
	FileIOChannel.ID channel = enumerator.next();
	channelManager.addChannel(channel);

	AbstractChannelWriterOutputView output = null;
	int bytesInLastBuffer;
	int blockCount;
	try {
		numSpillFiles++;
		output = FileChannelUtil.createOutputView(ioManager, channel, compressionEnable,
				compressionCodecFactory, compressionBlockSize, pageSize);
		buffer.writeToOutput(output);
		spillInBytes += output.getNumBytes();
		spillInCompressedBytes += output.getNumCompressedBytes();
		bytesInLastBuffer = output.close();
		blockCount = output.getBlockCount();
		LOG.info("here spill the {}th kv external buffer data with {} bytes and {} compressed bytes",
				numSpillFiles, spillInBytes, spillInCompressedBytes);
	} catch (IOException e) {
		if (output != null) {
			output.close();
			output.getChannel().deleteChannel();
		}
		throw e;
	}
	channelIDs.add(new ChannelWithMeta(channel, blockCount, bytesInLastBuffer));
}
 

public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}