org.apache.flink.runtime.io.disk.FileChannelInputView Java Examples

public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
 

public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
 

public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}