org.apache.flink.table.functions.UserDefinedFunction Java Examples

FunctionMappingExtractor(
		DataTypeFactory typeFactory,
		Class<? extends UserDefinedFunction> function,
		String methodName,
		SignatureExtraction signatureExtraction,
		@Nullable ResultExtraction accumulatorExtraction,
		ResultExtraction outputExtraction,
		MethodVerification verification) {
	this.typeFactory = typeFactory;
	this.function = function;
	this.methodName = methodName;
	this.signatureExtraction = signatureExtraction;
	this.accumulatorExtraction = accumulatorExtraction;
	this.outputExtraction = outputExtraction;
	this.verification = verification;
}
 

private static FunctionArgumentTemplate extractDataTypeArgument(
		DataTypeFactory typeFactory,
		Class<? extends UserDefinedFunction> function,
		Method method,
		int paramPos) {
	final DataType type = DataTypeExtractor.extractFromMethodParameter(
		typeFactory,
		function,
		method,
		paramPos);
	// unwrap data type in case of varargs
	if (method.isVarArgs() && paramPos == method.getParameterCount() - 1) {
		// for ARRAY
		if (type instanceof CollectionDataType) {
			return FunctionArgumentTemplate.of(((CollectionDataType) type).getElementDataType());
		}
		// special case for varargs that have been misinterpreted as BYTES
		else {
			return FunctionArgumentTemplate.of(DataTypes.TINYINT().notNull().bridgedTo(byte.class));
		}
	}
	return FunctionArgumentTemplate.of(type);
}
 

private static Optional<UserDefinedFunction> extractUserDefinedFunction(FunctionDefinition functionDefinition) {
	if (functionDefinition instanceof UserDefinedFunction) {
		return Optional.of((UserDefinedFunction) functionDefinition);
	} else if (functionDefinition instanceof ScalarFunctionDefinition) {
		return Optional.ofNullable(((ScalarFunctionDefinition) functionDefinition).getScalarFunction());
	} else if (functionDefinition instanceof AggregateFunctionDefinition) {
		return Optional.ofNullable(((AggregateFunctionDefinition) functionDefinition).getAggregateFunction());
	} else if (functionDefinition instanceof TableFunctionDefinition) {
		return Optional.ofNullable(((TableFunctionDefinition) functionDefinition).getTableFunction());
	} else if (functionDefinition instanceof TableAggregateFunctionDefinition) {
		return Optional.ofNullable(
			((TableAggregateFunctionDefinition) functionDefinition).getTableAggregateFunction()
		);
	}
	return Optional.empty();
}
 

@SuppressWarnings("unchecked")
private void validateAndPrepareFunction(CatalogFunction function) throws ClassNotFoundException {
	// If the input is instance of UserDefinedFunction, it means it uses the new type inference.
	// In this situation the UDF have not been validated and cleaned, so we need to validate it
	// and clean its closure here.
	// If the input is instance of `ScalarFunctionDefinition`, `TableFunctionDefinition` and so on,
	// it means it uses the old type inference. We assume that they have been validated before being
	// wrapped.
	if (function instanceof InlineCatalogFunction &&
		((InlineCatalogFunction) function).getDefinition() instanceof UserDefinedFunction) {

		FunctionDefinition definition = ((InlineCatalogFunction) function).getDefinition();
		UserDefinedFunctionHelper.prepareInstance(config, (UserDefinedFunction) definition);
	} else if (function.getFunctionLanguage() == FunctionLanguage.JAVA) {
		ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
		UserDefinedFunctionHelper.validateClass(
			(Class<? extends UserDefinedFunction>) contextClassLoader.loadClass(function.getClassName()));
	}
}
 

@SuppressWarnings("RedundantIfStatement")
private boolean verifyFunctionKind(
		@Nullable SqlFunctionCategory category,
		FunctionDefinition definition) {

	// for now, we don't allow other functions than user-defined ones
	// all built-in functions need to be mapped to Calcite's SqlFunctions
	if (!(definition instanceof UserDefinedFunction)) {
		return false;
	}

	// it would be nice to give a more meaningful exception when a scalar function is used instead
	// of a table function and vice versa, but we can do that only once FLIP-51 is implemented

	if (definition.getKind() == FunctionKind.SCALAR &&
			(category == SqlFunctionCategory.USER_DEFINED_FUNCTION || category == SqlFunctionCategory.SYSTEM)) {
		return true;
	} else if (definition.getKind() == FunctionKind.TABLE &&
			(category == SqlFunctionCategory.USER_DEFINED_TABLE_FUNCTION || category == SqlFunctionCategory.SYSTEM)) {
		return true;
	}

	// aggregate function are not supported, because the code generator is not ready yet

	return false;
}
 

public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create table sources & sinks.
	tableSources = new HashMap<>();
	tableSinks = new HashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new HashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}
 

private void registerFunctions() {
	Map<String, FunctionDefinition> functions = new LinkedHashMap<>();
	environment.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(
			entry.getDescriptor(),
			classLoader,
			false,
			getTableEnvironment().getConfig().getConfiguration());
		functions.put(name, function);
	});
	registerFunctions(functions);
}
 

private static List<FunctionArgumentTemplate> extractArgumentTemplates(
		DataTypeFactory typeFactory,
		Class<? extends UserDefinedFunction> function,
		Method method,
		int offset) {
	return IntStream.range(offset, method.getParameterCount())
		.mapToObj(i ->
			// check for input group before start extracting a data type
			tryExtractInputGroupArgument(method, i)
				.orElseGet(() -> extractDataTypeArgument(typeFactory, function, method, i)))
		.collect(Collectors.toList());
}
 

/**
 * Extraction that uses a generic type variable for producing a {@link FunctionResultTemplate}.
 */
static ResultExtraction createGenericResultExtraction(Class<? extends UserDefinedFunction> baseClass, int genericPos) {
	return (extractor, method) -> {
		final DataType dataType = DataTypeExtractor.extractFromGeneric(
			extractor.typeFactory,
			baseClass,
			genericPos,
			extractor.function);
		return FunctionResultTemplate.of(dataType);
	};
}
 

@Override
public boolean isGeneric() {
	if (functionLanguage == FunctionLanguage.PYTHON) {
		return true;
	}
	try {
		Class c = Class.forName(className);
		if (UserDefinedFunction.class.isAssignableFrom(c)) {
			return true;
		}
	} catch (ClassNotFoundException e) {
		throw new RuntimeException(String.format("Can't resolve udf class %s", className), e);
	}
	return false;
}
 

private static String createInlineFunctionName(FunctionDefinition functionDefinition) {
	final Optional<UserDefinedFunction> userDefinedFunction = extractUserDefinedFunction(functionDefinition);

	return userDefinedFunction.map(UserDefinedFunction::functionIdentifier)
		.orElseThrow(() -> new TableException(String.format(
			"Unsupported function definition: %s. Only user defined functions are supported as inline functions.",
			functionDefinition)));
}
 

@Override
public void createTemporarySystemFunction(String name, UserDefinedFunction functionInstance) {
	functionCatalog.registerTemporarySystemFunction(
		name,
		functionInstance,
		false);
}
 

@Override
public void createFunction(String path, Class<? extends UserDefinedFunction> functionClass, boolean ignoreIfExists) {
	final UnresolvedIdentifier unresolvedIdentifier = parser.parseIdentifier(path);
	functionCatalog.registerCatalogFunction(
		unresolvedIdentifier,
		functionClass,
		ignoreIfExists);
}
 

/**
 * Validates and cleans an inline, unregistered {@link UserDefinedFunction}.
 */
private FunctionDefinition prepareInlineUserDefinedFunction(FunctionDefinition definition) {
	if (definition instanceof ScalarFunctionDefinition) {
		final ScalarFunctionDefinition sf = (ScalarFunctionDefinition) definition;
		UserDefinedFunctionHelper.prepareInstance(resolutionContext.configuration(), sf.getScalarFunction());
		return new ScalarFunctionDefinition(
			sf.getName(),
			sf.getScalarFunction());
	} else if (definition instanceof TableFunctionDefinition) {
		final TableFunctionDefinition tf = (TableFunctionDefinition) definition;
		UserDefinedFunctionHelper.prepareInstance(resolutionContext.configuration(), tf.getTableFunction());
		return new TableFunctionDefinition(
			tf.getName(),
			tf.getTableFunction(),
			tf.getResultType());
	} else if (definition instanceof AggregateFunctionDefinition) {
		final AggregateFunctionDefinition af = (AggregateFunctionDefinition) definition;
		UserDefinedFunctionHelper.prepareInstance(resolutionContext.configuration(), af.getAggregateFunction());
		return new AggregateFunctionDefinition(
			af.getName(),
			af.getAggregateFunction(),
			af.getResultTypeInfo(),
			af.getAccumulatorTypeInfo());
	} else if (definition instanceof TableAggregateFunctionDefinition) {
		final TableAggregateFunctionDefinition taf = (TableAggregateFunctionDefinition) definition;
		UserDefinedFunctionHelper.prepareInstance(resolutionContext.configuration(), taf.getTableAggregateFunction());
		return new TableAggregateFunctionDefinition(
			taf.getName(),
			taf.getTableAggregateFunction(),
			taf.getResultTypeInfo(),
			taf.getAccumulatorTypeInfo());
	} else if (definition instanceof UserDefinedFunction) {
		UserDefinedFunctionHelper.prepareInstance(
			resolutionContext.configuration(),
			(UserDefinedFunction) definition);
	}
	return definition;
}
 

@Override
public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) {
	final UnresolvedIdentifier unresolvedIdentifier = parser.parseIdentifier(path);
	functionCatalog.registerTemporaryCatalogFunction(
		unresolvedIdentifier,
		functionInstance,
		false);
}
 

private void registerFunctions(Map<String, FunctionDefinition> functions) {
	if (tableEnv instanceof StreamTableEnvironment) {
		StreamTableEnvironment streamTableEnvironment = (StreamTableEnvironment) tableEnv;
		functions.forEach((k, v) -> {
			// Blink planner uses FLIP-65 functions for scalar and table functions
			// aggregate functions still use the old type inference
			if (environment.getExecution().isBlinkPlanner()) {
				if (v instanceof ScalarFunction || v instanceof TableFunction) {
					streamTableEnvironment.createTemporarySystemFunction(k, (UserDefinedFunction) v);
				} else if (v instanceof AggregateFunction) {
					streamTableEnvironment.registerFunction(k, (AggregateFunction<?, ?>) v);
				} else {
					throw new SqlExecutionException("Unsupported function type: " + v.getClass().getName());
				}
			}
			// legacy
			else {
				if (v instanceof ScalarFunction) {
					streamTableEnvironment.registerFunction(k, (ScalarFunction) v);
				} else if (v instanceof AggregateFunction) {
					streamTableEnvironment.registerFunction(k, (AggregateFunction<?, ?>) v);
				} else if (v instanceof TableFunction) {
					streamTableEnvironment.registerFunction(k, (TableFunction<?>) v);
				} else {
					throw new SqlExecutionException("Unsupported function type: " + v.getClass().getName());
				}
			}
		});
	} else {
		BatchTableEnvironment batchTableEnvironment = (BatchTableEnvironment) tableEnv;
		functions.forEach((k, v) -> {
			if (v instanceof ScalarFunction) {
				batchTableEnvironment.registerFunction(k, (ScalarFunction) v);
			} else if (v instanceof AggregateFunction) {
				batchTableEnvironment.registerFunction(k, (AggregateFunction<?, ?>) v);
			} else if (v instanceof TableFunction) {
				batchTableEnvironment.registerFunction(k, (TableFunction<?>) v);
			} else {
				throw new SqlExecutionException("Unsupported function type: " + v.getClass().getName());
			}
		});
	}
}
 

@Override
public void createTemporaryFunction(String path, Class<? extends UserDefinedFunction> functionClass) {
	final UserDefinedFunction functionInstance = UserDefinedFunctionHelper.instantiateFunction(functionClass);
	createTemporaryFunction(path, functionInstance);
}
 

@Override
public void createFunction(String path, Class<? extends UserDefinedFunction> functionClass) {
	createFunction(path, functionClass, false);
}
 

@Override
public void createTemporarySystemFunction(String name, Class<? extends UserDefinedFunction> functionClass) {
	final UserDefinedFunction functionInstance = UserDefinedFunctionHelper.instantiateFunction(functionClass);
	createTemporarySystemFunction(name, functionInstance);
}
 

/**
 * Retrieve global templates from function class.
 */
static Set<FunctionTemplate> extractGlobalFunctionTemplates(
		DataTypeFactory typeFactory,
		Class<? extends UserDefinedFunction> function) {
	return asFunctionTemplates(typeFactory, collectAnnotationsOfClass(FunctionHint.class, function));
}
 

Class<? extends UserDefinedFunction> getFunction() {
	return function;
}
 

public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create catalogs
	catalogs = new LinkedHashMap<>();
	mergedEnv.getCatalogs().forEach((name, entry) ->
		catalogs.put(name, createCatalog(name, entry.asMap(), classLoader))
	);

	// create table sources & sinks.
	tableSources = new LinkedHashMap<>();
	tableSinks = new LinkedHashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new LinkedHashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}
 

/**
 * A call to an unregistered, inline function.
 *
 * <p>For functions that have been registered before and are identified by a name, use
 * {@link #call(String, Object...)}.
 */
public static ApiExpression call(UserDefinedFunction function, Object... arguments) {
	return apiCall(function, arguments);
}
 

/**
 * A call to an unregistered, inline function.
 *
 * <p>For functions that have been registered before and are identified by a name, use
 * {@link #call(String, Object...)}.
 */
public static ApiExpression call(Class<? extends UserDefinedFunction> function, Object... arguments) {
	final UserDefinedFunction functionInstance = UserDefinedFunctionHelper.instantiateFunction(function);
	return apiCall(functionInstance, arguments);
}
 

/**
 * Registers a {@link UserDefinedFunction} class as a temporary system function.
 *
 * <p>Compared to {@link #createTemporaryFunction(String, Class)}, system functions are identified
 * by a global name that is independent of the current catalog and current database. Thus, this method
 * allows to extend the set of built-in system functions like {@code TRIM}, {@code ABS}, etc.
 *
 * <p>Temporary functions can shadow permanent ones. If a permanent function under a given name exists,
 * it will be inaccessible in the current session. To make the permanent function available again
 * one can drop the corresponding temporary system function.
 *
 * @param name The name under which the function will be registered globally.
 * @param functionClass The function class containing the implementation.
 */
@Experimental
void createTemporarySystemFunction(String name, Class<? extends UserDefinedFunction> functionClass);
 

/**
 * Registers a {@link UserDefinedFunction} instance as a temporary system function.
 *
 * <p>Compared to {@link #createTemporarySystemFunction(String, Class)}, this method takes a function
 * instance that might have been parameterized before (e.g. through its constructor). This might be
 * useful for more interactive sessions. Make sure that the instance is {@link Serializable}.
 *
 * <p>Compared to {@link #createTemporaryFunction(String, UserDefinedFunction)}, system functions are
 * identified by a global name that is independent of the current catalog and current database. Thus,
 * this method allows to extend the set of built-in system functions like {@code TRIM}, {@code ABS}, etc.
 *
 * <p>Temporary functions can shadow permanent ones. If a permanent function under a given name exists,
 * it will be inaccessible in the current session. To make the permanent function available again
 * one can drop the corresponding temporary system function.
 *
 * @param name The name under which the function will be registered globally.
 * @param functionInstance The (possibly pre-configured) function instance containing the implementation.
 */
@Experimental
void createTemporarySystemFunction(String name, UserDefinedFunction functionInstance);
 

/**
 * Registers a {@link UserDefinedFunction} class as a catalog function in the given path.
 *
 * <p>Compared to system functions with a globally defined name, catalog functions are always (implicitly
 * or explicitly) identified by a catalog and database.
 *
 * <p>There must not be another function (temporary or permanent) registered under the same path.
 *
 * @param path The path under which the function will be registered.
 *             See also the {@link TableEnvironment} class description for the format of the path.
 * @param functionClass The function class containing the implementation.
 */
@Experimental
void createFunction(String path, Class<? extends UserDefinedFunction> functionClass);
 

/**
 * Registers a {@link UserDefinedFunction} class as a catalog function in the given path.
 *
 * <p>Compared to system functions with a globally defined name, catalog functions are always (implicitly
 * or explicitly) identified by a catalog and database.
 *
 * @param path The path under which the function will be registered.
 *             See also the {@link TableEnvironment} class description for the format of the path.
 * @param functionClass The function class containing the implementation.
 * @param ignoreIfExists If a function exists under the given path and this flag is set, no operation
 *                       is executed. An exception is thrown otherwise.
 */
@Experimental
void createFunction(String path, Class<? extends UserDefinedFunction> functionClass, boolean ignoreIfExists);
 

/**
 * Registers a {@link UserDefinedFunction} class as a temporary catalog function.
 *
 * <p>Compared to {@link #createTemporarySystemFunction(String, Class)} with a globally defined name,
 * catalog functions are always (implicitly or explicitly) identified by a catalog and database.
 *
 * <p>Temporary functions can shadow permanent ones. If a permanent function under a given name exists,
 * it will be inaccessible in the current session. To make the permanent function available again
 * one can drop the corresponding temporary function.
 *
 * @param path The path under which the function will be registered.
 *             See also the {@link TableEnvironment} class description for the format of the path.
 * @param functionClass The function class containing the implementation.
 */
@Experimental
void createTemporaryFunction(String path, Class<? extends UserDefinedFunction> functionClass);
 

/**
 * Registers a {@link UserDefinedFunction} instance as a temporary catalog function.
 *
 * <p>Compared to {@link #createTemporaryFunction(String, Class)}, this method takes a function instance
 * that might have been parameterized before (e.g. through its constructor). This might be useful for more
 * interactive sessions. Make sure that the instance is {@link Serializable}.
 *
 * <p>Compared to {@link #createTemporarySystemFunction(String, UserDefinedFunction)} with a globally
 * defined name, catalog functions are always (implicitly or explicitly) identified by a catalog and database.
 *
 * <p>Temporary functions can shadow permanent ones. If a permanent function under a given name exists,
 * it will be inaccessible in the current session. To make the permanent function available again
 * one can drop the corresponding temporary function.
 *
 * @param path The path under which the function will be registered.
 *             See also the {@link TableEnvironment} class description for the format of the path.
 * @param functionInstance The (possibly pre-configured) function instance containing the implementation.
 */
@Experimental
void createTemporaryFunction(String path, UserDefinedFunction functionInstance);