Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializer#createInstance()

@Override
public void run() {
	final MutableObjectIterator<T> input = this.input;
	final TypeSerializer<T> serializer = this.serializer;
	final SpillingBuffer buffer = this.buffer;
	
	try {
		T record = serializer.createInstance();
		
		while (this.running && ((record = input.next(record)) != null)) {
			serializer.serialize(record, buffer);
		}
		
		TempBarrier.this.writingDone();
	}
	catch (Throwable t) {
		TempBarrier.this.setException(t);
	}
}
 

private SpillingResettableIterator(Iterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.instance = serializer.createInstance();
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 

@Override
@Test
public void testInstantiate() {
	try {
		TypeSerializer<T> serializer = getSerializer();

		T instance = serializer.createInstance();
		assertNotNull("The created instance must not be null.", instance);

		Class<T> type = getTypeClass();
		assertNotNull("The test is corrupt: type class is null.", type);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

public ReusingMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 

@Override
public void run() {
	final MutableObjectIterator<T> input = this.input;
	final TypeSerializer<T> serializer = this.serializer;
	final SpillingBuffer buffer = this.buffer;
	
	try {
		T record = serializer.createInstance();
		
		while (this.running && ((record = input.next(record)) != null)) {
			serializer.serialize(record, buffer);
		}
		
		TempBarrier.this.writingDone();
	}
	catch (Throwable t) {
		TempBarrier.this.setException(t);
	}
}
 

public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
 

public BlockResettableMutableObjectIterator(MemoryManager memoryManager,
		MutableObjectIterator<T> input,	TypeSerializer<T> serializer,
		int numMemoryPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	super(serializer, memoryManager, numMemoryPages, ownerTask);
	
	this.input = input;
	this.leftOverRecord = serializer.createInstance();
	this.leftOverReturned = true;
}
 

public BlockResettableMutableObjectIterator(MemoryManager memoryManager,
		MutableObjectIterator<T> input,	TypeSerializer<T> serializer,
		int numMemoryPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	super(serializer, memoryManager, numMemoryPages, ownerTask);
	
	this.input = input;
	this.leftOverRecord = serializer.createInstance();
	this.leftOverReturned = true;
}
 

public ReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.nextBuildSideObject = serializer2.createInstance();
	this.tempBuildSideRecord = serializer2.createInstance();

	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1, comparator1, pairComparator,
		memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
 

public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
 

public ReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.nextBuildSideObject = serializer2.createInstance();
	this.tempBuildSideRecord = serializer2.createInstance();

	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1, comparator1, pairComparator,
		memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

public ReusingBuildFirstHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V2, V1> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer2;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.nextBuildSideObject = serializer1.createInstance();
	this.tempBuildSideRecord = serializer1.createInstance();

	this.hashJoin = getHashJoin(serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 

public ReusingMutableToRegularIteratorWrapper(MutableObjectIterator<T> source,
		TypeSerializer<T> serializer) {
	this.source = source;
	this.current = serializer.createInstance();
	this.next = serializer.createInstance();
}
 

@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 

@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 

/**
 * Creates the reading thread. The reading thread simply reads the data off the input and puts it
 * into the buffer where it will be sorted.
 * <p>
 * The returned thread is not yet started.
 * 
 * @param exceptionHandler
 *        The handler for exceptions in the thread.
 * @param reader
 *        The reader from which the thread reads.
 * @param queues
 *        The queues through which the thread communicates with the other threads.
 * @param parentTask
 *        The task at which the thread registers itself (for profiling purposes).
 * @param serializer
 *        The serializer used to serialize records.
 * @param startSpillingBytes
 *        The number of bytes after which the reader thread will send the notification to
 *        start the spilling.
 *        
 * @return The thread that reads data from an input, writes it into sort buffers and puts 
 *         them into a queue.
 */
protected ThreadBase<E> getReadingThread(ExceptionHandler<IOException> exceptionHandler,
		MutableObjectIterator<E> reader, CircularQueues<E> queues, 
		LargeRecordHandler<E> largeRecordHandler, AbstractInvokable parentTask,
		TypeSerializer<E> serializer, long startSpillingBytes)
{
	return new ReadingThread<E>(exceptionHandler, reader, queues, largeRecordHandler, 
			serializer.createInstance(),parentTask, startSpillingBytes);
}
 

/**
 * Creates the reading thread. The reading thread simply reads the data off the input and puts it
 * into the buffer where it will be sorted.
 * <p>
 * The returned thread is not yet started.
 * 
 * @param exceptionHandler
 *        The handler for exceptions in the thread.
 * @param reader
 *        The reader from which the thread reads.
 * @param queues
 *        The queues through which the thread communicates with the other threads.
 * @param parentTask
 *        The task at which the thread registers itself (for profiling purposes).
 * @param serializer
 *        The serializer used to serialize records.
 * @param startSpillingBytes
 *        The number of bytes after which the reader thread will send the notification to
 *        start the spilling.
 *        
 * @return The thread that reads data from an input, writes it into sort buffers and puts 
 *         them into a queue.
 */
protected ThreadBase<E> getReadingThread(ExceptionHandler<IOException> exceptionHandler,
		MutableObjectIterator<E> reader, CircularQueues<E> queues, 
		LargeRecordHandler<E> largeRecordHandler, AbstractInvokable parentTask,
		TypeSerializer<E> serializer, long startSpillingBytes)
{
	return new ReadingThread<E>(exceptionHandler, reader, queues, largeRecordHandler, 
			serializer.createInstance(),parentTask, startSpillingBytes);
}