org.apache.flink.api.common.functions.CrossFunction Java Examples

@Override
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size());
	
	TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig);
	TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN1 element1 : inputData1) {
		for (IN2 element2 : inputData2) {
			IN1 copy1 = inSerializer1.copy(element1);
			IN2 copy2 = inSerializer2.copy(element2);
			OUT o = function.cross(copy1, copy2);
			result.add(outSerializer.copy(o));
		}
	}

	FunctionUtils.closeFunction(function);
	return result;
}
 

@Override
protected CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

	String name = getName() != null ? getName() : "Cross at " + defaultName;
	// create operator
	CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> po =
			new CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>>(function,
					new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
					name);

	po.setFirstInput(input1);
	po.setSecondInput(input2);
	po.setParallelism(getParallelism());
	po.setCrossHint(hint);

	return po;
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) crossInterface,
		CrossFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@Override
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size());
	
	TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig);
	TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN1 element1 : inputData1) {
		for (IN2 element2 : inputData2) {
			IN1 copy1 = inSerializer1.copy(element1);
			IN2 copy2 = inSerializer2.copy(element2);
			OUT o = function.cross(copy1, copy2);
			result.add(outSerializer.copy(o));
		}
	}

	FunctionUtils.closeFunction(function);
	return result;
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) crossInterface,
		CrossFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) crossInterface,
		CrossFunction.class,
		0,
		1,
		2,
		NO_INDEX,
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 

@Override
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size());
	
	TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig);
	TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN1 element1 : inputData1) {
		for (IN2 element2 : inputData2) {
			IN1 copy1 = inSerializer1.copy(element1);
			IN2 copy2 = inSerializer2.copy(element2);
			OUT o = function.cross(copy1, copy2);
			result.add(outSerializer.copy(o));
		}
	}

	FunctionUtils.closeFunction(function);
	return result;
}
 

@Override
protected CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

	String name = getName() != null ? getName() : "Cross at " + defaultName;
	// create operator
	CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> po =
			new CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>>(function,
					new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
					name);

	po.setFirstInput(input1);
	po.setSecondInput(input2);
	po.setParallelism(getParallelism());
	po.setCrossHint(hint);

	return po;
}
 

@Override
protected CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

	String name = getName() != null ? getName() : "Cross at " + defaultName;
	// create operator
	CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> po =
			new CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>>(function,
					new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
					name);

	po.setFirstInput(input1);
	po.setSecondInput(input2);
	po.setParallelism(getParallelism());
	po.setCrossHint(hint);

	return po;
}
 

private static DataSet <Tuple2 <Integer, DenseVector>>
computeDissimilarityVectors(DataSet <Tuple2 <Integer, DenseVector>> inputVectors) {
	return inputVectors
		.cross(inputVectors)
		.with(
			new CrossFunction <Tuple2 <Integer, DenseVector>, Tuple2 <Integer, DenseVector>, Tuple3 <Integer,
				Integer, Double>>() {
				@Override
				public Tuple3 <Integer, Integer, Double> cross(Tuple2 <Integer, DenseVector> val1,
															   Tuple2 <Integer, DenseVector> val2)
					throws Exception {
					return Tuple3.of(val1.f0, val2.f0, MatVecOp.minus(val1.f1, val2.f1).normL2Square());
				}
			})
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GroupReduceFunction <Tuple3 <Integer, Integer, Double>, Tuple2 <Integer, DenseVector>>() {
			@Override
			public void reduce(Iterable <Tuple3 <Integer, Integer, Double>> values,
							   Collector <Tuple2 <Integer, DenseVector>> out) throws Exception {
				int id = -1;
				List <Double> d = new ArrayList <>();
				for (Tuple3 <Integer, Integer, Double> v : values) {
					id = v.f0;
					d.add(v.f2);
				}
				DenseVector denseVector = new DenseVector(d.size());
				for (int i = 0; i < d.size(); i++) {
					denseVector.set(i, d.get(i));
				}
				out.collect(Tuple2.of(id, denseVector));
			}
		})
		.name("computeDissimilarityVectors");
}
 

public CrossOperator(DataSet<I1> input1, DataSet<I2> input2,
						CrossFunction<I1, I2, OUT> function,
						TypeInformation<OUT> returnType,
						CrossHint hint,
						String defaultName) {
	super(input1, input2, returnType);

	this.function = function;
	this.defaultName = defaultName;
	this.hint = hint;

	UdfOperatorUtils.analyzeDualInputUdf(this, CrossFunction.class, defaultName, function, null, null);
}
 

public CrossOperator(DataSet<I1> input1, DataSet<I2> input2,
						CrossFunction<I1, I2, OUT> function,
						TypeInformation<OUT> returnType,
						CrossHint hint,
						String defaultName) {
	super(input1, input2, returnType);

	this.function = function;
	this.defaultName = defaultName;
	this.hint = hint;
}
 

public CrossOperator(DataSet<I1> input1, DataSet<I2> input2,
						CrossFunction<I1, I2, OUT> function,
						TypeInformation<OUT> returnType,
						CrossHint hint,
						String defaultName) {
	super(input1, input2, returnType);

	this.function = function;
	this.defaultName = defaultName;
	this.hint = hint;
}
 

@Override
protected CrossFunction<I1, I2, OUT> getFunction() {
	return function;
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type)
{
	return getCrossReturnTypes(crossInterface, in1Type, in2Type, null, false);
}
 

@Override
public void setup(TaskContext<CrossFunction<T1, T2, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;
}
 

@Override
public Class<CrossFunction<T1, T2, OT>> getStubType() {
	@SuppressWarnings("unchecked")
	final Class<CrossFunction<T1, T2, OT>> clazz = (Class<CrossFunction<T1, T2, OT>>) (Class<?>) CrossFunction.class;
	return clazz;
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type)
{
	return getCrossReturnTypes(crossInterface, in1Type, in2Type, null, false);
}
 

@Override
public Class<CrossFunction<T1, T2, OT>> getStubType() {
	@SuppressWarnings("unchecked")
	final Class<CrossFunction<T1, T2, OT>> clazz = (Class<CrossFunction<T1, T2, OT>>) (Class<?>) CrossFunction.class;
	return clazz;
}
 

@Override
public void setup(TaskContext<CrossFunction<T1, T2, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;
}
 

@Override
protected CrossFunction<I1, I2, OUT> getFunction() {
	return function;
}
 

@Override
public Class<CrossFunction<T1, T2, OT>> getStubType() {
	@SuppressWarnings("unchecked")
	final Class<CrossFunction<T1, T2, OT>> clazz = (Class<CrossFunction<T1, T2, OT>>) (Class<?>) CrossFunction.class;
	return clazz;
}
 

@Override
public void setup(TaskContext<CrossFunction<T1, T2, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;
}
 

@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type)
{
	return getCrossReturnTypes(crossInterface, in1Type, in2Type, null, false);
}
 

public UdfAnalyzer(Class<?> baseClass, Class<?> udfClass, String externalUdfName,
		TypeInformation<?> in1Type, TypeInformation<?> in2Type,
		TypeInformation<?> outType, Keys<?> keys1, Keys<?> keys2,
		boolean throwErrorExceptions) {

	baseClassMethod = baseClass.getDeclaredMethods()[0];
	this.udfClass = udfClass;
	this.externalUdfName = externalUdfName;
	this.internalUdfClassName = Type.getInternalName(udfClass);
	this.in1Type = in1Type;
	this.in2Type = in2Type;
	this.outType = outType;
	this.keys1 = keys1;
	this.keys2 = keys2;
	this.throwErrorExceptions = throwErrorExceptions;

	if (baseClass == CoGroupFunction.class) {
		hasCollector = true;
		isBinary = true;
		isIterableInput = true;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == CrossFunction.class) {
		hasCollector = false;
		isBinary = true;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == FlatJoinFunction.class) {
		hasCollector = true;
		isBinary = true;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == FlatMapFunction.class) {
		hasCollector = true;
		isBinary = false;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == GroupReduceFunction.class) {
		hasCollector = true;
		isBinary = false;
		isIterableInput = true;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = false;
	}
	else if (baseClass == JoinFunction.class) {
		hasCollector = false;
		isBinary = true;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == MapFunction.class) {
		hasCollector = false;
		isBinary = false;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == ReduceFunction.class) {
		hasCollector = false;
		isBinary = false;
		isIterableInput = false;
		isReduceFunction = true;
		isFilterFunction = false;
		iteratorTrueAssumptionApplied = true;
	}
	else if (baseClass == FilterFunction.class) {
		hasCollector = false;
		isBinary = false;
		isIterableInput = false;
		isReduceFunction = false;
		isFilterFunction = true;
		iteratorTrueAssumptionApplied = true;
	}
	// TODO MapPartitionFunction, GroupCombineFunction and CombineFunction not implemented yet
	else {
		throw new UnsupportedOperationException("Unsupported operator.");
	}
	if (hasCollector) {
		collectorValues = new ArrayList<TaggedValue>();
	}
	else {
		collectorValues = null;
	}
	hints = new ArrayList<String>();
}
 

@Override
protected CrossFunction<I1, I2, OUT> getFunction() {
	return function;
}
 

/**
 * Finalizes a Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.
 *
 * <p>Each CrossFunction call returns exactly one element.
 *
 * @param function The CrossFunction that is called for each pair of crossed elements.
 * @return An CrossOperator that represents the crossed result DataSet
 *
 * @see CrossFunction
 * @see DataSet
 */
public <R> CrossOperator<I1, I2, R> with(CrossFunction<I1, I2, R> function) {
	if (function == null) {
		throw new NullPointerException("Cross function must not be null.");
	}
	TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, getInput1().getType(), getInput2().getType(),
			super.getDefaultName(), true);
	return new CrossOperator<I1, I2, R>(getInput1(), getInput2(), clean(function), returnType,
			getCrossHint(), Utils.getCallLocationName());
}
 

/**
 * Finalizes a Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.
 *
 * <p>Each CrossFunction call returns exactly one element.
 *
 * @param function The CrossFunction that is called for each pair of crossed elements.
 * @return An CrossOperator that represents the crossed result DataSet
 *
 * @see CrossFunction
 * @see DataSet
 */
public <R> CrossOperator<I1, I2, R> with(CrossFunction<I1, I2, R> function) {
	if (function == null) {
		throw new NullPointerException("Cross function must not be null.");
	}
	TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, getInput1().getType(), getInput2().getType(),
			super.getDefaultName(), true);
	return new CrossOperator<I1, I2, R>(getInput1(), getInput2(), clean(function), returnType,
			getCrossHint(), Utils.getCallLocationName());
}
 

/**
 * Finalizes a Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.
 *
 * <p>Each CrossFunction call returns exactly one element.
 *
 * @param function The CrossFunction that is called for each pair of crossed elements.
 * @return An CrossOperator that represents the crossed result DataSet
 *
 * @see CrossFunction
 * @see DataSet
 */
public <R> CrossOperator<I1, I2, R> with(CrossFunction<I1, I2, R> function) {
	if (function == null) {
		throw new NullPointerException("Cross function must not be null.");
	}
	TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, getInput1().getType(), getInput2().getType(),
			super.getDefaultName(), true);
	return new CrossOperator<I1, I2, R>(getInput1(), getInput2(), clean(function), returnType,
			getCrossHint(), Utils.getCallLocationName());
}