org.apache.flink.api.common.typeutils.TypeSerializerFactory Java Examples

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 

/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 

protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 

@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 

private void setTypeSerializerFactory(TypeSerializerFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 

@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final ReduceFunction<IT> red = BatchTask.instantiateUserCode(this.config, userCodeClassLoader, ReduceFunction.class);
	this.reducer = red;
	FunctionUtils.setFunctionRuntimeContext(red, getUdfRuntimeContext());

	TypeSerializerFactory<IT> serializerFactory = this.config.getInputSerializer(0, userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ChainedAllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 

protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new SpillingThread(exceptionHandler, queues, parentTask,
		memoryManager, ioManager, serializerFactory.getSerializer(), comparator, sortReadMemory, writeMemory, maxFileHandles);
}
 

@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index < 0 || index >= this.driver.getNumberOfInputs()) {
		throw new IndexOutOfBoundsException();
	}

	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory = (TypeSerializerFactory<X>) this.inputSerializers[index];
	return serializerFactory;
}
 

@Override
@SuppressWarnings("unchecked")
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	switch (index) {
	case 0:
		return (TypeSerializerFactory<X>) this.serializer1;
	case 1:
		return (TypeSerializerFactory<X>) this.serializer2;
	default:
		throw new RuntimeException();
	}
}
 

@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index != 0) {
		throw new IllegalArgumentException();
	}
	
	@SuppressWarnings("unchecked")
	TypeSerializer<X> ser = (TypeSerializer<X>) inputSerializer;
	return new RuntimeSerializerFactory<X>(ser, (Class<X>) ser.createInstance().getClass());
}
 

@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 

/**
 * Creates all the serializers and iterators for the broadcast inputs.
 */
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
	this.broadcastInputSerializers = new TypeSerializerFactory<?>[numBroadcastInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();

	for (int i = 0; i < numBroadcastInputs; i++) {
		//  ---------------- create the serializer first ---------------------
		final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, userCodeClassLoader);
		this.broadcastInputSerializers[i] = serializerFactory;
	}
}
 

protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

/**
 * @return output serializer of this task
 */
private TypeSerializer<OT> getOutputSerializer() {
	TypeSerializerFactory<OT> serializerFactory;

	if ((serializerFactory = getLastTasksConfig().getOutputSerializer(getUserCodeClassLoader())) ==
			null) {
		throw new RuntimeException("Missing output serializer for workset update.");
	}

	return serializerFactory.getSerializer();
}
 

public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 

@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 

private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 

private void setTypeSerializerFactory(TypeSerializerFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 

@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index != 0) {
		throw new IllegalArgumentException();
	}
	
	@SuppressWarnings("unchecked")
	TypeSerializer<X> ser = (TypeSerializer<X>) inputSerializer;
	return new RuntimeSerializerFactory<X>(ser, (Class<X>) ser.createInstance().getClass());
}
 

private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 

@Override
@SuppressWarnings("unchecked")
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	switch (index) {
	case 0:
		return (TypeSerializerFactory<X>) this.serializer1;
	case 1:
		return (TypeSerializerFactory<X>) this.serializer2;
	default:
		throw new RuntimeException();
	}
}
 

/**
 * Creates all the serializers and iterators for the broadcast inputs.
 */
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
	this.broadcastInputSerializers = new TypeSerializerFactory<?>[numBroadcastInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();

	for (int i = 0; i < numBroadcastInputs; i++) {
		//  ---------------- create the serializer first ---------------------
		final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, userCodeClassLoader);
		this.broadcastInputSerializers[i] = serializerFactory;
	}
}
 

protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

protected UnilateralSortMerger(MemoryManager memoryManager,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, memoryManager.allocatePages(parentTask, memoryManager.computeNumberOfPages(memoryFraction)),
			ioManager, input, parentTask, serializerFactory, comparator,
			numSortBuffers, maxNumFileHandles, startSpillingFraction, noSpillingMemory, handleLargeRecords,
			objectReuseEnabled);
}
 

@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 

public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int maxNumFileHandles, float startSpillingFraction,
		boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, -1, maxNumFileHandles, startSpillingFraction, handleLargeRecords, objectReuseEnabled);
}
 

public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 

protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new SpillingThread(exceptionHandler, queues, parentTask,
		memoryManager, ioManager, serializerFactory.getSerializer(), comparator, sortReadMemory, writeMemory, maxFileHandles);
}