org.apache.flink.api.common.functions.Function Java Examples

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) crossInterface,
		CrossFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getFlatJoinReturnTypes(FlatJoinFunction<IN1, IN2, OUT> joinInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) joinInterface,
		FlatJoinFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

public static void restoreFunctionState(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);

	while (true) {

		if (tryRestoreFunction(context, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can restore the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getFlatJoinReturnTypes(FlatJoinFunction<IN1, IN2, OUT> joinInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) joinInterface,
		FlatJoinFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@SuppressWarnings("unchecked")
public static <T> void setOutputType(
		Function userFunction,
		TypeInformation<T> outTypeInfo,
		ExecutionConfig executionConfig) {

	Preconditions.checkNotNull(outTypeInfo);
	Preconditions.checkNotNull(executionConfig);

	while (true) {
		if (trySetOutputType(userFunction, outTypeInfo, executionConfig)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}

	}
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(CoGroupFunction<IN1, IN2, OUT> coGroupInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) coGroupInterface,
		CoGroupFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getJoinReturnTypes(JoinFunction<IN1, IN2, OUT> joinInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) joinInterface,
		JoinFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) crossInterface,
		CrossFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

private static String generateFunctionName(Function function) {
	Class<? extends Function> functionClass = function.getClass();
	if (functionClass.isAnonymousClass()) {
		// getSimpleName returns an empty String for anonymous classes
		Type[] interfaces = functionClass.getInterfaces();
		if (interfaces.length == 0) {
			// extends an existing class (like RichMapFunction)
			Class<?> functionSuperClass = functionClass.getSuperclass();
			return functionSuperClass.getSimpleName() + functionClass.getName().substring(functionClass.getEnclosingClass().getName().length());
		} else {
			// implements a Function interface
			Class<?> functionInterface = functionClass.getInterfaces()[0];
			return functionInterface.getSimpleName() + functionClass.getName().substring(functionClass.getEnclosingClass().getName().length());
		}
	} else {
		return functionClass.getSimpleName();
	}
}
 

public <S extends Function, OT> Class<? extends Driver<S, OT>> getDriver() {
	final String className = this.config.getString(DRIVER_CLASS, null);
	if (className == null) {
		throw new CorruptConfigurationException("The pact driver class is missing.");
	}
	
	try {
		@SuppressWarnings("unchecked")
		final Class<Driver<S, OT>> pdClazz = (Class<Driver<S, OT>>) (Class<?>) Driver.class;
		return Class.forName(className).asSubclass(pdClazz);
	} catch (ClassNotFoundException cnfex) {
		throw new CorruptConfigurationException("The given driver class cannot be found.");
	} catch (ClassCastException ccex) {
		throw new CorruptConfigurationException("The given driver class does not implement the pact driver interface.");
	}
}
 

@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 

@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 

public static void restoreFunctionState(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);

	while (true) {

		if (tryRestoreFunction(context, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can restore the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

@SuppressWarnings("unchecked")
public static <T> void setOutputType(
		Function userFunction,
		TypeInformation<T> outTypeInfo,
		ExecutionConfig executionConfig) {

	Preconditions.checkNotNull(outTypeInfo);
	Preconditions.checkNotNull(executionConfig);

	while (true) {
		if (trySetOutputType(userFunction, outTypeInfo, executionConfig)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}

	}
}
 

public <S extends Function, OT> Class<? extends Driver<S, OT>> getDriver() {
	final String className = this.config.getString(DRIVER_CLASS, null);
	if (className == null) {
		throw new CorruptConfigurationException("The pact driver class is missing.");
	}
	
	try {
		@SuppressWarnings("unchecked")
		final Class<Driver<S, OT>> pdClazz = (Class<Driver<S, OT>>) (Class<?>) Driver.class;
		return Class.forName(className).asSubclass(pdClazz);
	} catch (ClassNotFoundException cnfex) {
		throw new CorruptConfigurationException("The given driver class cannot be found.");
	} catch (ClassCastException ccex) {
		throw new CorruptConfigurationException("The given driver class does not implement the pact driver interface.");
	}
}
 

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		ListState<Serializable> listState = context.getOperatorStateStore().
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}
 

@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getGroupCombineReturnTypes(GroupCombineFunction<IN, OUT> combineInterface, TypeInformation<IN> inType,
																		String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) combineInterface,
		GroupCombineFunction.class,
		0,
		1,
		new int[]{1, 0},
		inType,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getGroupReduceReturnTypes(GroupReduceFunction<IN, OUT> groupReduceInterface, TypeInformation<IN> inType,
		String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) groupReduceInterface,
		GroupReduceFunction.class,
		0,
		1,
		new int[]{1, 0},
		inType,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getMapReturnTypes(MapFunction<IN, OUT> mapInterface, TypeInformation<IN> inType,
		String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) mapInterface,
		MapFunction.class,
		0,
		1,
		NO_INDEX,
		inType,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getFlatMapReturnTypes(FlatMapFunction<IN, OUT> flatMapInterface, TypeInformation<IN> inType,
		String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) flatMapInterface,
		FlatMapFunction.class,
		0,
		1,
		new int[]{1, 0},
		inType,
		functionName,
		allowMissing);
}
 

/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) foldInterface,
		FoldFunction.class,
		0,
		1,
		NO_INDEX,
		inType,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN, OUT> TypeInformation<OUT> getMapPartitionReturnTypes(MapPartitionFunction<IN, OUT> mapPartitionInterface, TypeInformation<IN> inType,
		String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) mapPartitionInterface,
		MapPartitionFunction.class,
		0,
		1,
		new int[]{1, 0},
		inType,
		functionName,
		allowMissing);
}
 

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}
 

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}
 

@SuppressWarnings("unchecked")
private static  <T> boolean trySetOutputType(
		Function userFunction,
		TypeInformation<T> outTypeInfo,
		ExecutionConfig executionConfig) {

	Preconditions.checkNotNull(outTypeInfo);
	Preconditions.checkNotNull(executionConfig);

	if (OutputTypeConfigurable.class.isAssignableFrom(userFunction.getClass())) {
		((OutputTypeConfigurable<T>) userFunction).setOutputType(outTypeInfo, executionConfig);
		return true;
	}
	return false;
}
 

private static String generateOperatorName(
		WindowAssigner<?, ?> assigner,
		Trigger<?, ?> trigger,
		@Nullable Evictor<?, ?> evictor,
		Function function1,
		@Nullable Function function2) {
	return "Window(" +
		assigner + ", " +
		trigger.getClass().getSimpleName() + ", " +
		(evictor == null ? "" : (evictor.getClass().getSimpleName() + ", ")) +
		generateFunctionName(function1) +
		(function2 == null ? "" : (", " + generateFunctionName(function2))) +
		")";
}
 

private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		ListState<Serializable> listState = context.getOperatorStateStore().
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}
 

@SuppressWarnings("unchecked")
private static  <T> boolean trySetOutputType(
		Function userFunction,
		TypeInformation<T> outTypeInfo,
		ExecutionConfig executionConfig) {

	Preconditions.checkNotNull(outTypeInfo);
	Preconditions.checkNotNull(executionConfig);

	if (OutputTypeConfigurable.class.isAssignableFrom(userFunction.getClass())) {
		((OutputTypeConfigurable<T>) userFunction).setOutputType(outTypeInfo, executionConfig);
		return true;
	}
	return false;
}