Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.IOManager#createBlockChannelWriter()

public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 

public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 

public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 

public static BlockChannelWriter<MemorySegment> createBlockChannelWriter(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelWriter(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelWriter(channel, bufferReturnQueue);
	}
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 

/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 

public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 

public static BlockChannelWriter<MemorySegment> createBlockChannelWriter(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelWriter(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelWriter(channel, bufferReturnQueue);
	}
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	if (isInMemory()) {
		return;
	}
	// ATTENTION: The following lines are duplicated code from finalizeBuildPhase
	this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
	this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 

public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	if (isInMemory()) {
		return;
	}
	// ATTENTION: The following lines are duplicated code from finalizeBuildPhase
	this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
	this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
}
 

@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 

public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	if (isInMemory()) {
		return;
	}
	// ATTENTION: The following lines are duplicated code from finalizeBuildPhase
	this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
	this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
}
 

/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}