org.apache.flink.runtime.memory.ListMemorySegmentSource Java Examples

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

/**
 * Creates a new partition, in memory, with one buffer. 
 * 
 * @param serializer Serializer for T.
 * @param partitionNumber The number of the partition.
 * @param memSource memory pool
 * @param pageSize segment size in bytes
 * @param pageSizeInBits
 */
public InMemoryPartition(TypeSerializer<T> serializer, int partitionNumber,
		ListMemorySegmentSource memSource, int pageSize, int pageSizeInBits)
{
	this.overflowSegments = new MemorySegment[2];
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	this.serializer = serializer;
	this.partitionPages = new ArrayList<MemorySegment>(64);
	this.availableMemory = memSource;
	
	this.partitionNumber = partitionNumber;
	
	// add the first segment
	this.partitionPages.add(memSource.nextSegment());
	// empty partitions have no garbage
	this.compacted = true;
	
	this.pageSize = pageSize;
	
	this.pageSizeInBits = pageSizeInBits;
	
	this.writeView = new WriteView(this.partitionPages, memSource, pageSize, pageSizeInBits);
	this.readView = new ReadView(this.partitionPages, pageSize, pageSizeInBits);
}
 

/**
 * Creates a new partition, in memory, with one buffer. 
 * 
 * @param serializer Serializer for T.
 * @param partitionNumber The number of the partition.
 * @param memSource memory pool
 * @param pageSize segment size in bytes
 * @param pageSizeInBits
 */
public InMemoryPartition(TypeSerializer<T> serializer, int partitionNumber,
		ListMemorySegmentSource memSource, int pageSize, int pageSizeInBits)
{
	this.overflowSegments = new MemorySegment[2];
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	this.serializer = serializer;
	this.partitionPages = new ArrayList<MemorySegment>(64);
	this.availableMemory = memSource;
	
	this.partitionNumber = partitionNumber;
	
	// add the first segment
	this.partitionPages.add(memSource.nextSegment());
	// empty partitions have no garbage
	this.compacted = true;
	
	this.pageSize = pageSize;
	
	this.pageSizeInBits = pageSizeInBits;
	
	this.writeView = new WriteView(this.partitionPages, memSource, pageSize, pageSizeInBits);
	this.readView = new ReadView(this.partitionPages, pageSize, pageSizeInBits);
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 

/**
 * Creates a new partition, in memory, with one buffer. 
 * 
 * @param serializer Serializer for T.
 * @param partitionNumber The number of the partition.
 * @param memSource memory pool
 * @param pageSize segment size in bytes
 * @param pageSizeInBits
 */
public InMemoryPartition(TypeSerializer<T> serializer, int partitionNumber,
		ListMemorySegmentSource memSource, int pageSize, int pageSizeInBits)
{
	this.overflowSegments = new MemorySegment[2];
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	this.serializer = serializer;
	this.partitionPages = new ArrayList<MemorySegment>(64);
	this.availableMemory = memSource;
	
	this.partitionNumber = partitionNumber;
	
	// add the first segment
	this.partitionPages.add(memSource.nextSegment());
	// empty partitions have no garbage
	this.compacted = true;
	
	this.pageSize = pageSize;
	
	this.pageSizeInBits = pageSizeInBits;
	
	this.writeView = new WriteView(this.partitionPages, memSource, pageSize, pageSizeInBits);
	this.readView = new ReadView(this.partitionPages, pageSize, pageSizeInBits);
}
 

private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

private void createPartitions(int numPartitions) {
	this.partitions.clear();
	
	ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
	
	for (int i = 0; i < numPartitions; i++) {
		this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
	}
	this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits);
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

private void createPartitions(int numPartitions) {
	this.partitions.clear();
	
	ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
	
	for (int i = 0; i < numPartitions; i++) {
		this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
	}
	this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits);
}
 

private InMemoryBuffer(AbstractRowSerializer serializer) {
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<>(memory);
	// serializer has states, so we must duplicate
	this.serializer = (AbstractRowSerializer) serializer.duplicate();
	this.recordBufferSegments = new ArrayList<>(memory.size());
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
			new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordCount = 0;
}
 

public TempBarrier(AbstractInvokable owner, MutableObjectIterator<T> input, TypeSerializerFactory<T> serializerFactory,
		MemoryManager memManager, IOManager ioManager, int numPages) throws MemoryAllocationException
{
	this.serializer = serializerFactory.getSerializer();
	this.memManager = memManager;
	
	this.memory = new ArrayList<MemorySegment>(numPages);
	memManager.allocatePages(owner, this.memory, numPages);
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(this.memory), memManager.getPageSize());
	this.tempWriter = new TempWritingThread(input, serializerFactory.getSerializer(), this.buffer);
}
 

private void createPartitions(int numPartitions) {
	this.partitions.clear();
	
	ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
	
	for (int i = 0; i < numPartitions; i++) {
		this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
	}
	this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits);
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 

public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}