org.apache.flink.runtime.io.disk.SimpleCollectingOutputView Java Examples

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

/**
 * Create a memory sorter in `insert` way.
 */
public static BinaryInMemorySortBuffer createBuffer(
		NormalizedKeyComputer normalizedKeyComputer,
		AbstractRowSerializer<BaseRow> inputSerializer,
		BinaryRowSerializer serializer,
		RecordComparator comparator,
		List<MemorySegment> memory) throws IOException {
	checkArgument(memory.size() >= MIN_REQUIRED_BUFFERS);
	int totalNumBuffers = memory.size();
	ListMemorySegmentPool pool = new ListMemorySegmentPool(memory);
	ArrayList<MemorySegment> recordBufferSegments = new ArrayList<>(16);
	return new BinaryInMemorySortBuffer(
			normalizedKeyComputer, inputSerializer, serializer, comparator, recordBufferSegments,
			new SimpleCollectingOutputView(recordBufferSegments, pool, pool.pageSize()),
			pool, totalNumBuffers);
}
 

private void writeKVToBuffer(
		BinaryRowSerializer keySerializer,
		BinaryRowSerializer valueSerializer,
		SimpleCollectingOutputView out,
		List<Integer> expecteds,
		int length) throws IOException {
	Random random = new Random();
	int stringLength = 30;
	for (int i = 0; i < length; i++) {
		BinaryRow key = randomRow(random, stringLength);
		BinaryRow val = key.copy();
		val.setInt(0, val.getInt(0) * -3 + 177);
		expecteds.add(key.getInt(0));
		keySerializer.serializeToPages(key, out);
		valueSerializer.serializeToPages(val, out);
	}
}
 

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

private void writeKVToBuffer(
		BinaryRowDataSerializer keySerializer,
		BinaryRowDataSerializer valueSerializer,
		SimpleCollectingOutputView out,
		List<Integer> expecteds,
		int length) throws IOException {
	Random random = new Random();
	int stringLength = 30;
	for (int i = 0; i < length; i++) {
		BinaryRowData key = randomRow(random, stringLength);
		BinaryRowData val = key.copy();
		val.setInt(0, val.getInt(0) * -3 + 177);
		expecteds.add(key.getInt(0));
		keySerializer.serializeToPages(key, out);
		valueSerializer.serializeToPages(val, out);
	}
}
 

private BinaryInMemorySortBuffer(
		NormalizedKeyComputer normalizedKeyComputer,
		AbstractRowSerializer<BaseRow> inputSerializer,
		BinaryRowSerializer serializer,
		RecordComparator comparator,
		ArrayList<MemorySegment> recordBufferSegments,
		SimpleCollectingOutputView recordCollector,
		MemorySegmentPool pool,
		int totalNumBuffers) throws IOException {
	super(normalizedKeyComputer, serializer, comparator, recordBufferSegments, pool);
	this.inputSerializer = inputSerializer;
	this.recordBufferSegments = recordBufferSegments;
	this.recordCollector = recordCollector;
	this.totalNumBuffers = totalNumBuffers;
}
 

private InMemoryBuffer(AbstractRowSerializer serializer) {
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<>(memory);
	// serializer has states, so we must duplicate
	this.serializer = (AbstractRowSerializer) serializer.duplicate();
	this.recordBufferSegments = new ArrayList<>(memory.size());
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
			new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordCount = 0;
}
 

@Test
public void test() throws Exception {
	BufferedKVExternalSorter sorter =
			new BufferedKVExternalSorter(
					ioManager, keySerializer, valueSerializer, computer, comparator,
					PAGE_SIZE, conf);
	TestMemorySegmentPool pool = new TestMemorySegmentPool(PAGE_SIZE);
	List<Integer> expected = new ArrayList<>();
	for (int i = 0; i < spillNumber; i++) {
		ArrayList<MemorySegment> segments = new ArrayList<>();
		SimpleCollectingOutputView out =
				new SimpleCollectingOutputView(segments, pool, PAGE_SIZE);
		writeKVToBuffer(
				keySerializer, valueSerializer,
				out,
				expected,
				recordNumberPerFile);
		sorter.sortAndSpill(segments, recordNumberPerFile, pool);
	}
	Collections.sort(expected);
	MutableObjectIterator<Tuple2<BinaryRow, BinaryRow>> iterator = sorter.getKVIterator();
	Tuple2<BinaryRow, BinaryRow> kv =
			new Tuple2<>(keySerializer.createInstance(), valueSerializer.createInstance());
	int count = 0;
	while ((kv = iterator.next(kv)) != null) {
		Assert.assertEquals((int) expected.get(count), kv.f0.getInt(0));
		Assert.assertEquals(expected.get(count) * -3 + 177, kv.f1.getInt(0));
		count++;
	}
	Assert.assertEquals(expected.size(), count);
	sorter.close();
}
 

/**
 * Create a memory sorter in `insert` way.
 */
public static BinaryInMemorySortBuffer createBuffer(
		NormalizedKeyComputer normalizedKeyComputer,
		AbstractRowDataSerializer<RowData> inputSerializer,
		BinaryRowDataSerializer serializer,
		RecordComparator comparator,
		MemorySegmentPool memoryPool) {
	checkArgument(memoryPool.freePages() >= MIN_REQUIRED_BUFFERS);
	int totalNumBuffers = memoryPool.freePages();
	ArrayList<MemorySegment> recordBufferSegments = new ArrayList<>(16);
	return new BinaryInMemorySortBuffer(
			normalizedKeyComputer, inputSerializer, serializer, comparator, recordBufferSegments,
			new SimpleCollectingOutputView(recordBufferSegments, memoryPool, memoryPool.pageSize()),
			memoryPool, totalNumBuffers);
}
 

private BinaryInMemorySortBuffer(
		NormalizedKeyComputer normalizedKeyComputer,
		AbstractRowDataSerializer<RowData> inputSerializer,
		BinaryRowDataSerializer serializer,
		RecordComparator comparator,
		ArrayList<MemorySegment> recordBufferSegments,
		SimpleCollectingOutputView recordCollector,
		MemorySegmentPool pool,
		int totalNumBuffers) {
	super(normalizedKeyComputer, serializer, comparator, recordBufferSegments, pool);
	this.inputSerializer = inputSerializer;
	this.recordBufferSegments = recordBufferSegments;
	this.recordCollector = recordCollector;
	this.totalNumBuffers = totalNumBuffers;
}
 

private InMemoryBuffer(AbstractRowDataSerializer serializer) {
	// serializer has states, so we must duplicate
	this.serializer = (AbstractRowDataSerializer) serializer.duplicate();
	this.recordBufferSegments = new ArrayList<>();
	this.recordCollector = new SimpleCollectingOutputView(
			this.recordBufferSegments, pool, segmentSize);
	this.recordCount = 0;
}
 

@Test
public void test() throws Exception {
	BufferedKVExternalSorter sorter =
			new BufferedKVExternalSorter(
					ioManager, keySerializer, valueSerializer, computer, comparator,
					PAGE_SIZE, conf);
	TestMemorySegmentPool pool = new TestMemorySegmentPool(PAGE_SIZE);
	List<Integer> expected = new ArrayList<>();
	for (int i = 0; i < spillNumber; i++) {
		ArrayList<MemorySegment> segments = new ArrayList<>();
		SimpleCollectingOutputView out =
				new SimpleCollectingOutputView(segments, pool, PAGE_SIZE);
		writeKVToBuffer(
				keySerializer, valueSerializer,
				out,
				expected,
				recordNumberPerFile);
		sorter.sortAndSpill(segments, recordNumberPerFile, pool);
	}
	Collections.sort(expected);
	MutableObjectIterator<Tuple2<BinaryRowData, BinaryRowData>> iterator = sorter.getKVIterator();
	Tuple2<BinaryRowData, BinaryRowData> kv =
			new Tuple2<>(keySerializer.createInstance(), valueSerializer.createInstance());
	int count = 0;
	while ((kv = iterator.next(kv)) != null) {
		Assert.assertEquals((int) expected.get(count), kv.f0.getInt(0));
		Assert.assertEquals(expected.get(count) * -3 + 177, kv.f1.getInt(0));
		count++;
	}
	Assert.assertEquals(expected.size(), count);
	sorter.close();
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 

RecordArea() {
	this.outView = new SimpleCollectingOutputView(segments, memoryPool, segmentSize);
	this.inView = new RandomAccessInputView(segments, segmentSize);
}
 

RecordArea() {
	this.outView = new SimpleCollectingOutputView(segments, new RecordAreaMemorySource(), segmentSize);
	this.inView = new RandomAccessInputView(segments, segmentSize);
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}