Java Code Examples for org.apache.flink.api.common.operators.Keys#SelectorFunctionKeys

public <K> SortedGrouping(DataSet<T> set, Keys<T> keys, Keys.SelectorFunctionKeys<T, K> keySelector, Order order) {
	super(set, keys);

	if (!(this.keys instanceof Keys.SelectorFunctionKeys)) {
		throw new InvalidProgramException("Sorting on KeySelector keys only works with KeySelector grouping.");
	}
	TypeInformation<?> sortKeyType = keySelector.getKeyType();
	if (!sortKeyType.isSortKeyType()) {
		throw new InvalidProgramException("Key type " + sortKeyType + " is not sortable.");
	}

	this.groupSortKeyPositions = keySelector.computeLogicalKeyPositions();
	for (int i = 0; i < groupSortKeyPositions.length; i++) {
		groupSortKeyPositions[i] += this.keys.getNumberOfKeyFields();
	}

	this.groupSortSelectorFunctionKey = keySelector;
	this.groupSortOrders = new Order[groupSortKeyPositions.length];
	Arrays.fill(this.groupSortOrders, order);
}
 

public PlanLeftUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		int[] key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<I2> typeInfo2) {

	super(
			new TupleLeftUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, I2, OUT>(
					typeInfoWithKey1,
					typeInfo2,
					resultType),
			key1.computeLogicalKeyPositions(),
			key2,
			name);
}
 

public PlanRightUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		int[] key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<I1> typeInfo1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleRightUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<I1, Tuple2<K, I2>, OUT>(
					typeInfo1,
					typeInfoWithKey2,
					resultType),
			key1,
			key2.computeLogicalKeyPositions(),
			name);
}
 

public <K> SortedGrouping(DataSet<T> set, Keys<T> keys, Keys.SelectorFunctionKeys<T, K> keySelector, Order order) {
	super(set, keys);

	if (!(this.keys instanceof Keys.SelectorFunctionKeys)) {
		throw new InvalidProgramException("Sorting on KeySelector keys only works with KeySelector grouping.");
	}
	TypeInformation<?> sortKeyType = keySelector.getKeyType();
	if (!sortKeyType.isSortKeyType()) {
		throw new InvalidProgramException("Key type " + sortKeyType + " is not sortable.");
	}

	this.groupSortKeyPositions = keySelector.computeLogicalKeyPositions();
	for (int i = 0; i < groupSortKeyPositions.length; i++) {
		groupSortKeyPositions[i] += this.keys.getNumberOfKeyFields();
	}

	this.groupSortSelectorFunctionKey = keySelector;
	this.groupSortOrders = new Order[groupSortKeyPositions.length];
	Arrays.fill(this.groupSortOrders, order);
}
 

public <K> SortedGrouping(DataSet<T> set, Keys<T> keys, Keys.SelectorFunctionKeys<T, K> keySelector, Order order) {
	super(set, keys);

	if (!(this.keys instanceof Keys.SelectorFunctionKeys)) {
		throw new InvalidProgramException("Sorting on KeySelector keys only works with KeySelector grouping.");
	}
	TypeInformation<?> sortKeyType = keySelector.getKeyType();
	if (!sortKeyType.isSortKeyType()) {
		throw new InvalidProgramException("Key type " + sortKeyType + " is not sortable.");
	}

	this.groupSortKeyPositions = keySelector.computeLogicalKeyPositions();
	for (int i = 0; i < groupSortKeyPositions.length; i++) {
		groupSortKeyPositions[i] += this.keys.getNumberOfKeyFields();
	}

	this.groupSortSelectorFunctionKey = keySelector;
	this.groupSortOrders = new Order[groupSortKeyPositions.length];
	Arrays.fill(this.groupSortOrders, order);
}
 

private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
 

public <K> SortPartitionOperator(DataSet<T> dataSet, Keys.SelectorFunctionKeys<T, K> sortKey, Order sortOrder, String sortLocationName) {
	this(dataSet, sortLocationName);
	this.useKeySelector = true;

	ensureSortableKey(sortKey);

	keys.add(sortKey);
	orders.add(sortOrder);
}
 

public PlanUnwrappingReduceGroupOperator(
	GroupReduceFunction<IN, OUT> udf,
	Keys.SelectorFunctionKeys<IN, K> key,
	String name,
	TypeInformation<OUT> outType,
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey,
	boolean combinable) {
	super(
		combinable ?
			new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K>(udf) :
			new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf),
		new UnaryOperatorInformation<>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name);

	super.setCombinable(combinable);
}
 

private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
 

public <K> SortPartitionOperator(DataSet<T> dataSet, Keys.SelectorFunctionKeys<T, K> sortKey, Order sortOrder, String sortLocationName) {
	this(dataSet, sortLocationName);
	this.useKeySelector = true;

	ensureSortableKey(sortKey);

	keys.add(sortKey);
	orders.add(sortOrder);
}
 

private <K> void ensureSortableKey(Keys.SelectorFunctionKeys<T, K> sortKey) {
	if (!sortKey.getKeyType().isSortKeyType()) {
		throw new InvalidProgramException("Selected sort key is not a sortable type");
	}
}
 

protected Keys.SelectorFunctionKeys<T, ?> getSortSelectionFunctionKey() {
	return this.groupSortSelectorFunctionKey;
}
 

private <K> void ensureSortableKey(Keys.SelectorFunctionKeys<T, K> sortKey) {
	if (!sortKey.getKeyType().isSortKeyType()) {
		throw new InvalidProgramException("Selected sort key is not a sortable type");
	}
}
 

public PlanUnwrappingGroupCombineOperator(GroupCombineFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name,
											TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey) {
	super(new TupleUnwrappingGroupCombiner<IN, OUT, K>(udf),
			new UnaryOperatorInformation<Tuple2<K, IN>, OUT>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name);

}
 

/**
 * Sorts Pojos within a group on the specified field in the specified {@link Order}.
 *
 * <p><b>Note: Only groups of Tuple elements and Pojos can be sorted.</b>
 *
 * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(String, Order)} calls.
 *
 * @param field The Tuple or Pojo field on which the group is sorted.
 * @param order The Order in which the specified field is sorted.
 * @return A SortedGrouping with specified order of group element.
 *
 * @see Order
 */
public SortedGrouping<T> sortGroup(String field, Order order) {
	if (this.getKeys() instanceof Keys.SelectorFunctionKeys) {
		throw new InvalidProgramException("KeySelector grouping keys and field expression group-sorting keys cannot be used together.");
	}

	SortedGrouping<T> sg = new SortedGrouping<T>(this.inputDataSet, this.keys, field, order);
	sg.customPartitioner = getCustomPartitioner();
	return sg;
}
 

/**
 * Sorts elements within a group on a key extracted by the specified {@link org.apache.flink.api.java.functions.KeySelector}
 * in the specified {@link Order}.
 *
 * <p>Chaining {@link #sortGroup(KeySelector, Order)} calls is not supported.
 *
 * @param keySelector The KeySelector with which the group is sorted.
 * @param order The Order in which the extracted key is sorted.
 * @return A SortedGrouping with specified order of group element.
 *
 * @see Order
 */
public <K> SortedGrouping<T> sortGroup(KeySelector<T, K> keySelector, Order order) {
	if (!(this.getKeys() instanceof Keys.SelectorFunctionKeys)) {
		throw new InvalidProgramException("KeySelector group-sorting keys can only be used with KeySelector grouping keys.");
	}

	TypeInformation<K> keyType = TypeExtractor.getKeySelectorTypes(keySelector, this.inputDataSet.getType());
	SortedGrouping<T> sg = new SortedGrouping<T>(this.inputDataSet, this.keys, new Keys.SelectorFunctionKeys<T, K>(keySelector, this.inputDataSet.getType(), keyType), order);
	sg.customPartitioner = getCustomPartitioner();
	return sg;
}
 

/**
 * Sorts {@link org.apache.flink.api.java.tuple.Tuple} elements within a group on the specified field in the specified {@link Order}.
 *
 * <p><b>Note: Only groups of Tuple elements and Pojos can be sorted.</b>
 *
 * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(int, Order)} calls.
 *
 * @param field The Tuple field on which the group is sorted.
 * @param order The Order in which the specified Tuple field is sorted.
 * @return A SortedGrouping with specified order of group element.
 *
 * @see org.apache.flink.api.java.tuple.Tuple
 * @see Order
 */
public SortedGrouping<T> sortGroup(int field, Order order) {
	if (this.getKeys() instanceof Keys.SelectorFunctionKeys) {
		throw new InvalidProgramException("KeySelector grouping keys and field index group-sorting keys cannot be used together.");
	}

	SortedGrouping<T> sg = new SortedGrouping<T>(this.inputDataSet, this.keys, field, order);
	sg.customPartitioner = getCustomPartitioner();
	return sg;
}
 

/**
 * Partitions a DataSet using the specified KeySelector.
 *
 * <p><b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 *
 * @param keyExtractor The KeyExtractor with which the DataSet is hash-partitioned.
 * @return The partitioned DataSet.
 *
 * @see KeySelector
 */
public <K extends Comparable<K>> PartitionOperator<T> partitionByHash(KeySelector<T, K> keyExtractor) {
	final TypeInformation<K> keyType = TypeExtractor.getKeySelectorTypes(keyExtractor, getType());
	return new PartitionOperator<>(this, PartitionMethod.HASH, new Keys.SelectorFunctionKeys<>(clean(keyExtractor), this.getType(), keyType), Utils.getCallLocationName());
}
 

/**
 * Continues a Join transformation and defines a {@link KeySelector} function for the first join {@link DataSet}.
 *
 * <p>The KeySelector function is called for each element of the first DataSet and extracts a single
 * key value on which the DataSet is joined.
 *
 * @param keySelector The KeySelector function which extracts the key values from the DataSet on which it is joined.
 * @return An incomplete Join transformation.
 *           Call {@link org.apache.flink.api.java.operators.join.JoinOperatorSetsBase.JoinOperatorSetsPredicateBase#equalTo(int...)} or
 *           {@link org.apache.flink.api.java.operators.join.JoinOperatorSetsBase.JoinOperatorSetsPredicateBase#equalTo(KeySelector)}
 *           to continue the Join.
 *
 * @see KeySelector
 * @see DataSet
 */
public <K> JoinOperatorSetsPredicateBase where(KeySelector<I1, K> keySelector) {
	TypeInformation<K> keyType = TypeExtractor.getKeySelectorTypes(keySelector, input1.getType());
	return new JoinOperatorSetsPredicateBase(new Keys.SelectorFunctionKeys<>(keySelector, input1.getType(), keyType));
}
 

/**
 * Locally sorts the partitions of the DataSet on the extracted key in the specified order.
 * The DataSet can be sorted on multiple values by returning a tuple from the KeySelector.
 *
 * <p>Note that no additional sort keys can be appended to a KeySelector sort keys. To sort
 * the partitions by multiple values using KeySelector, the KeySelector must return a tuple
 * consisting of the values.
 *
 * @param keyExtractor The KeySelector function which extracts the key values from the DataSet
 *                     on which the DataSet is sorted.
 * @param order The order in which the DataSet is sorted.
 * @return The DataSet with sorted local partitions.
 */
public <K> SortPartitionOperator<T> sortPartition(KeySelector<T, K> keyExtractor, Order order) {
	final TypeInformation<K> keyType = TypeExtractor.getKeySelectorTypes(keyExtractor, getType());
	return new SortPartitionOperator<>(this, new Keys.SelectorFunctionKeys<>(clean(keyExtractor), getType(), keyType), order, Utils.getCallLocationName());
}