Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.IOManager#createBlockChannelReader()

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.IOManager#createBlockChannelReader() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileChannelUtil.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static BlockChannelReader<MemorySegment> createBlockChannelReader(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelReader(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelReader(channel, bufferReturnQueue);
	}
}
Example 2
Source File: FileChannelUtil.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static BlockChannelReader<MemorySegment> createBlockChannelReader(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelReader(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelReader(channel, bufferReturnQueue);
	}
}
Example 3
Source File: ChannelReaderInputViewIterator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel,  LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue,
		segments, freeMemTarget, accessors, numBlocks);
}
Example 4
Source File: SeekableFileChannelInputView.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
Example 5
Source File: FileChannelStreamsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCloseAndDeleteInputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
Example 6
Source File: FileChannelUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}
Example 7
Source File: ChannelReaderInputViewIterator.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel,  LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue,
		segments, freeMemTarget, accessors, numBlocks);
}
Example 8
Source File: SeekableFileChannelInputView.java    From flink with Apache License 2.0 5 votes vote down vote up 
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
Example 9
Source File: FileChannelUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}
Example 10
Source File: ChannelReaderInputViewIterator.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel,  LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue,
		segments, freeMemTarget, accessors, numBlocks);
}
Example 11
Source File: SeekableFileChannelInputView.java    From flink with Apache License 2.0 5 votes vote down vote up 
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}