org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys Java Examples

The following examples show how to use org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
		SelectorFunctionKeys<IN, ?> rawGroupingKey,
		SelectorFunctionKeys<IN, ?> rawSortingKeys,
		Ordering groupOrder,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKeys;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedGroupCombineOperator<>(function, groupingKey, sortingKey, name, outputType, typeInfoWithKey);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrder);

	return reducer;
}
Example #2
Source File: ReduceOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #3
Source File: GroupReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}
Example #4
Source File: DistinctOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #5
Source File: GroupReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
	SelectorFunctionKeys<IN, ?> rawGroupingKey,
	SelectorFunctionKeys<IN, ?> rawSortingKey,
	Ordering groupOrdering,
	GroupReduceFunction<IN, OUT> function,
	TypeInformation<OUT> outputType,
	String name,
	Operator<IN> input,
	boolean combinable) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedReduceGroupOperator<>(
			function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrdering);

	return reducer;
}
Example #6
Source File: JoinOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}
Example #7
Source File: CoGroupOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}
Example #8
Source File: CoGroupOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}
Example #9
Source File: ReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}
Example #10
Source File: ReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #11
Source File: DistinctOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #12
Source File: PartitionOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
Example #13
Source File: KeyFunctions.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover(
		org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey,
		SelectorFunctionKeys<T, K> key) {

	TypeInformation<T> inputType = key.getInputType();
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key);

	MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper =
			new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(
					new KeyRemovingMapper<T, K>(),
					new UnaryOperatorInformation<>(typeInfoWithKey, inputType),
					"Key Remover"
			);
	mapper.setInput(inputWithKey);
	mapper.setParallelism(inputWithKey.getParallelism());

	return mapper;
}
Example #14
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}
Example #15
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingGroupCombineOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingGroupCombineOperator<IN, OUT, K> reducer =
		new PlanUnwrappingGroupCombineOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);

	return reducer;
}
Example #16
Source File: GroupReduceOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}
Example #17
Source File: GroupReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}
Example #18
Source File: GroupReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}
Example #19
Source File: GroupReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
	SelectorFunctionKeys<IN, ?> rawGroupingKey,
	SelectorFunctionKeys<IN, ?> rawSortingKey,
	Ordering groupOrdering,
	GroupReduceFunction<IN, OUT> function,
	TypeInformation<OUT> outputType,
	String name,
	Operator<IN> input,
	boolean combinable) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedReduceGroupOperator<>(
			function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrdering);

	return reducer;
}
Example #20
Source File: JoinOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}
Example #21
Source File: CoGroupOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}
Example #22
Source File: CoGroupOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}
Example #23
Source File: ReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}
Example #24
Source File: ReduceOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #25
Source File: DistinctOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
Example #26
Source File: PartitionOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
Example #27
Source File: KeyFunctions.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover(
		org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey,
		SelectorFunctionKeys<T, K> key) {

	TypeInformation<T> inputType = key.getInputType();
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key);

	MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper =
			new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(
					new KeyRemovingMapper<T, K>(),
					new UnaryOperatorInformation<>(typeInfoWithKey, inputType),
					"Key Remover"
			);
	mapper.setInput(inputWithKey);
	mapper.setParallelism(inputWithKey.getParallelism());

	return mapper;
}
Example #28
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}
Example #29
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingGroupCombineOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingGroupCombineOperator<IN, OUT, K> reducer =
		new PlanUnwrappingGroupCombineOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);

	return reducer;
}
Example #30
Source File: GroupCombineOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
		SelectorFunctionKeys<IN, ?> rawGroupingKey,
		SelectorFunctionKeys<IN, ?> rawSortingKeys,
		Ordering groupOrder,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKeys;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedGroupCombineOperator<>(function, groupingKey, sortingKey, name, outputType, typeInfoWithKey);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrder);

	return reducer;
}