org.apache.flink.api.common.functions.util.FunctionUtils Java Examples

protected S initStub(Class<? super S> stubSuperClass) throws Exception {
	try {
		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
		S stub = config.<S>getStubWrapper(userCodeClassLoader).getUserCodeObject(stubSuperClass, userCodeClassLoader);
		// check if the class is a subclass, if the check is required
		if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
			throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" + 
					stubSuperClass.getName() + "' as is required.");
		}
		FunctionUtils.setFunctionRuntimeContext(stub, this.runtimeUdfContext);
		return stub;
	}
	catch (ClassCastException ccex) {
		throw new Exception("The stub class is not a proper subclass of " + stubSuperClass.getName(), ccex);
	}
}
 

@Override
protected List<OUT> executeOnCollections(List<IN> input, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<IN, OUT> function = userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(input.size());

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer);

	for (IN element : input) {
		IN inCopy = inSerializer.copy(element);
		function.flatMap(inCopy, resultCollector);
	}

	FunctionUtils.closeFunction(function);

	return result;
}
 

@Override
protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	MapPartitionFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<OUT> result = new ArrayList<OUT>(inputData.size() / 4);

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	CopyingIterator<IN> source = new CopyingIterator<IN>(inputData.iterator(), inSerializer);
	CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer);

	function.mapPartition(source, resultCollector);

	result.trimToSize();
	FunctionUtils.closeFunction(function);
	return result;
}
 

@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 

@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 

@Override
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size());
	
	TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig);
	TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN1 element1 : inputData1) {
		for (IN2 element2 : inputData2) {
			IN1 copy1 = inSerializer1.copy(element1);
			IN2 copy2 = inSerializer2.copy(element2);
			OUT o = function.cross(copy1, copy2);
			result.add(outSerializer.copy(o));
		}
	}

	FunctionUtils.closeFunction(function);
	return result;
}
 

@Override
public void close() throws Exception {
	try {
		finishBundle();
	} finally {
		Exception exception = null;

		try {
			super.close();
			if (function != null) {
				FunctionUtils.closeFunction(function);
			}
		} catch (InterruptedException interrupted) {
			exception = interrupted;

			Thread.currentThread().interrupt();
		} catch (Exception e) {
			exception = e;
		}

		if (exception != null) {
			LOG.warn("Errors occurred while closing the BundleOperator.", exception);
		}
	}
}
 

@Override
protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	MapFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<OUT> result = new ArrayList<OUT>(inputData.size());

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN element : inputData) {
		IN inCopy = inSerializer.copy(element);
		OUT out = function.map(inCopy);
		result.add(outSerializer.copy(out));
	}

	FunctionUtils.closeFunction(function);
	
	return result;
}
 

protected S initStub(Class<? super S> stubSuperClass) throws Exception {
	try {
		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
		S stub = config.<S>getStubWrapper(userCodeClassLoader).getUserCodeObject(stubSuperClass, userCodeClassLoader);
		// check if the class is a subclass, if the check is required
		if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
			throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" + 
					stubSuperClass.getName() + "' as is required.");
		}
		FunctionUtils.setFunctionRuntimeContext(stub, this.runtimeUdfContext);
		return stub;
	}
	catch (ClassCastException ccex) {
		throw new Exception("The stub class is not a proper subclass of " + stubSuperClass.getName(), ccex);
	}
}
 

@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyAllWindowContext<>(windowFunction);

}
 

@Override
public void close() throws Exception {
	super.close();
	if (fetcher != null) {
		FunctionUtils.closeFunction(fetcher);
	}
	if (collector != null) {
		FunctionUtils.closeFunction(collector);
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.fetcher = generatedFetcher.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(fetcher, getRuntimeContext());
	FunctionUtils.openFunction(fetcher, parameters);

	// try to compile the generated ResultFuture, fail fast if the code is corrupt.
	generatedResultFuture.compile(getRuntimeContext().getUserCodeClassLoader());

	// row converter is stateless which is thread-safe
	RowConverter rowConverter;
	if (fetcherReturnType instanceof RowTypeInfo) {
		rowConverter = (RowConverter) DataFormatConverters.getConverterForDataType(
				fromLegacyInfoToDataType(fetcherReturnType));
	} else if (fetcherReturnType instanceof BaseRowTypeInfo) {
		rowConverter = null;
	} else {
		throw new IllegalStateException("This should never happen, " +
			"currently fetcherReturnType can only be BaseRowTypeInfo or RowTypeInfo");
	}

	// asyncBufferCapacity + 1 as the queue size in order to avoid
	// blocking on the queue when taking a collector.
	this.resultFutureBuffer = new ArrayBlockingQueue<>(asyncBufferCapacity + 1);
	this.allResultFutures = new ArrayList<>();
	for (int i = 0; i < asyncBufferCapacity + 1; i++) {
		JoinedRowResultFuture rf = new JoinedRowResultFuture(
			resultFutureBuffer,
			createFetcherResultFuture(parameters),
			rowConverter,
			isLeftOuterJoin,
			rightRowTypeInfo.getArity());
		// add will throw exception immediately if the queue is full which should never happen
		resultFutureBuffer.add(rf);
		allResultFutures.add(rf);
	}
}
 

@Override
@SuppressWarnings("unchecked")
public void open(KeyGroupRangeInputSplit split) throws IOException {
	registry = new CloseableRegistry();

	final Environment environment = new SavepointEnvironment
		.Builder(getRuntimeContext(), split.getNumKeyGroups())
		.setSubtaskIndex(split.getSplitNumber())
		.setPrioritizedOperatorSubtaskState(split.getPrioritizedOperatorSubtaskState())
		.build();

	final StreamOperatorStateContext context = getStreamOperatorStateContext(environment);

	keyedStateBackend = (AbstractKeyedStateBackend<K>) context.keyedStateBackend();

	final DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, getRuntimeContext().getExecutionConfig());
	SavepointRuntimeContext ctx = new SavepointRuntimeContext(getRuntimeContext(), keyedStateStore);
	FunctionUtils.setFunctionRuntimeContext(userFunction, ctx);

	keys = getKeyIterator(ctx);

	final InternalTimerService<VoidNamespace> timerService = restoreTimerService(context);
	try {
		this.ctx = new Context(keyedStateBackend, timerService);
	} catch (Exception e) {
		throw new IOException("Failed to restore timer state", e);
	}
}
 

@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupReduceFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
	this.reducer = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 

public TableFunctionResultFuture<BaseRow> createFetcherResultFuture(Configuration parameters) throws Exception {
	TableFunctionResultFuture<BaseRow> resultFuture = generatedResultFuture.newInstance(
		getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(resultFuture, getRuntimeContext());
	FunctionUtils.openFunction(resultFuture, parameters);
	return resultFuture;
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.fetcher = generatedFetcher.newInstance(getRuntimeContext().getUserCodeClassLoader());
	this.collector = generatedCollector.newInstance(getRuntimeContext().getUserCodeClassLoader());

	FunctionUtils.setFunctionRuntimeContext(fetcher, getRuntimeContext());
	FunctionUtils.setFunctionRuntimeContext(collector, getRuntimeContext());
	FunctionUtils.openFunction(fetcher, parameters);
	FunctionUtils.openFunction(collector, parameters);

	this.nullRow = new GenericRow(tableFieldsCount);
	this.outRow = new JoinedRow();
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.calc = generatedCalc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(calc, getRuntimeContext());
	FunctionUtils.openFunction(calc, parameters);
	this.calcCollector = new CalcCollector(collector);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	for (IterativeCondition<T> nestedCondition : nestedConditions) {
		FunctionUtils.openFunction(nestedCondition, parameters);
	}
}
 

@Override
public TableFunctionResultFuture<BaseRow> createFetcherResultFuture(Configuration parameters) throws Exception {
	TableFunctionResultFuture<BaseRow> joinConditionCollector = super.createFetcherResultFuture(parameters);
	FlatMapFunction<BaseRow, BaseRow> calc = generatedCalc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(calc, getRuntimeContext());
	FunctionUtils.openFunction(calc, parameters);
	return new TemporalTableCalcResultFuture(calc, joinConditionCollector);
}
 

@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	}
	catch (Throwable t) {
	}
}
 

@Override
public void close() throws Exception {
	super.close();
	for (IterativeCondition<T> nestedCondition : nestedConditions) {
		FunctionUtils.closeFunction(nestedCondition);
	}
}
 

@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final MapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 

@Override
public void dispose() throws Exception {
	super.dispose();
	if (!functionsClosed) {
		functionsClosed = true;
		FunctionUtils.closeFunction(userFunction);
	}
}
 

@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyWindowContext<>(windowFunction);
}
 

@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	} catch (Throwable t) {
	}
}
 

@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;
	running = true;

	strategy = config.getDriverStrategy();

	reducer = BatchTask.instantiateUserCode(config, userCodeClassLoader, ReduceFunction.class);
	FunctionUtils.setFunctionRuntimeContext(reducer, getUdfRuntimeContext());
}
 

@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	} catch (Throwable t) {
	}
}
 

@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final MapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 

@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupReduceFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
	this.reducer = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 

@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.reducer);
	} catch (Throwable t) {
	}
}