org.apache.flink.runtime.io.disk.iomanager.HeaderlessChannelReaderInputView Java Examples

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.HeaderlessChannelReaderInputView. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ResettableExternalBuffer.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void newSpilledIterator(int channelID, long offset) throws IOException {
	ChannelWithMeta channel = spilledChannelIDs.get(channelID);
	currentChannelID = channelID;

	// close current reader first.
	closeCurrentFileReader();

	// calculate segment number
	int segmentNum = (int) (offset / segmentSize);
	long seekPosition = segmentNum * segmentSize;

	// new reader.
	this.fileReader = ioManager.createBlockChannelReader(channel.getChannel());
	if (offset > 0) {
		// seek to the beginning of that segment
		fileReader.seekToPosition(seekPosition);
	}
	ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(
		fileReader, getReadMemory(), channel.getBlockCount() - segmentNum,
		channel.getNumBytesInLastBlock(), false, offset - seekPosition
	);
	this.currentIterator = new BinaryRowChannelInputViewIterator(inView, binaryRowSerializer);
}
Example #2
Source File: ResettableExternalBuffer.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void newSpilledIterator(int channelID, long offset) throws IOException {
	ChannelWithMeta channel = spilledChannelIDs.get(channelID);
	currentChannelID = channelID;

	// close current reader first.
	closeCurrentFileReader();

	// calculate segment number
	int segmentNum = (int) (offset / segmentSize);
	long seekPosition = segmentNum * segmentSize;

	// new reader.
	this.fileReader = ioManager.createBlockChannelReader(channel.getChannel());
	if (offset > 0) {
		// seek to the beginning of that segment
		fileReader.seekToPosition(seekPosition);
	}
	ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(
		fileReader, getReadMemory(), channel.getBlockCount() - segmentNum,
		channel.getNumBytesInLastBlock(), false, offset - seekPosition
	);
	this.currentIterator = new BinaryRowChannelInputViewIterator(inView, binaryRowSerializer);
}
Example #3
Source File: BaseHybridHashTable.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
Example #4
Source File: FileChannelUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}
Example #5
Source File: BaseHybridHashTable.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
Example #6
Source File: FileChannelUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}
Example #7
Source File: SpillingBuffer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public DataInputView flip() throws IOException {
	// check whether this is the first flip and we need to add the current segment to the full ones
	if (getCurrentSegment() != null) {
		// first flip
		if (this.writer == null) {
			// in memory
			this.fullSegments.add(getCurrentSegment());
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.inMemInView = new RandomAccessInputView(this.fullSegments, this.segmentSize, this.numBytesInLastSegment);
		} else {
			// external: write the last segment and collect the memory back
			this.writer.writeBlock(this.getCurrentSegment());
			this.numMemorySegmentsInWriter++;
			
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.blockCount++;
			this.writer.close();
			for (int i = this.numMemorySegmentsInWriter; i > 0; i--) {
				this.fullSegments.add(this.writer.getNextReturnedBlock());
			}
			this.numMemorySegmentsInWriter = 0;
		}
		
		// make sure we cannot write more
		clear();
	}
	
	if (this.writer == null) {
		// in memory
		this.inMemInView.setReadPosition(0);
		return this.inMemInView;
	} else {
		// recollect memory from a previous view
		if (this.externalInView != null) {
			this.externalInView.close();
		}
		
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(this.writer.getChannelID());
		this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false);
		return this.externalInView;
	}
}
Example #8
Source File: SpillingBuffer.java    From flink with Apache License 2.0 4 votes vote down vote up 
public DataInputView flip() throws IOException {
	// check whether this is the first flip and we need to add the current segment to the full ones
	if (getCurrentSegment() != null) {
		// first flip
		if (this.writer == null) {
			// in memory
			this.fullSegments.add(getCurrentSegment());
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.inMemInView = new RandomAccessInputView(this.fullSegments, this.segmentSize, this.numBytesInLastSegment);
		} else {
			// external: write the last segment and collect the memory back
			this.writer.writeBlock(this.getCurrentSegment());
			this.numMemorySegmentsInWriter++;
			
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.blockCount++;
			this.writer.close();
			for (int i = this.numMemorySegmentsInWriter; i > 0; i--) {
				this.fullSegments.add(this.writer.getNextReturnedBlock());
			}
			this.numMemorySegmentsInWriter = 0;
		}
		
		// make sure we cannot write more
		clear();
	}
	
	if (this.writer == null) {
		// in memory
		this.inMemInView.setReadPosition(0);
		return this.inMemInView;
	} else {
		// recollect memory from a previous view
		if (this.externalInView != null) {
			this.externalInView.close();
		}
		
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(this.writer.getChannelID());
		this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false);
		return this.externalInView;
	}
}
Example #9
Source File: SpillingBuffer.java    From flink with Apache License 2.0 4 votes vote down vote up 
public DataInputView flip() throws IOException {
	// check whether this is the first flip and we need to add the current segment to the full ones
	if (getCurrentSegment() != null) {
		// first flip
		if (this.writer == null) {
			// in memory
			this.fullSegments.add(getCurrentSegment());
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.inMemInView = new RandomAccessInputView(this.fullSegments, this.segmentSize, this.numBytesInLastSegment);
		} else {
			// external: write the last segment and collect the memory back
			this.writer.writeBlock(this.getCurrentSegment());
			this.numMemorySegmentsInWriter++;
			
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.blockCount++;
			this.writer.close();
			for (int i = this.numMemorySegmentsInWriter; i > 0; i--) {
				this.fullSegments.add(this.writer.getNextReturnedBlock());
			}
			this.numMemorySegmentsInWriter = 0;
		}
		
		// make sure we cannot write more
		clear();
	}
	
	if (this.writer == null) {
		// in memory
		this.inMemInView.setReadPosition(0);
		return this.inMemInView;
	} else {
		// recollect memory from a previous view
		if (this.externalInView != null) {
			this.externalInView.close();
		}
		
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(this.writer.getChannelID());
		this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false);
		return this.externalInView;
	}
}