Java Code Examples for org.apache.flink.api.common.operators.base.SortPartitionOperatorBase#setParallelism()

The following examples show how to use org.apache.flink.api.common.operators.base.SortPartitionOperatorBase#setParallelism() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SortPartitionOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
Example 2
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
Example 3
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
Example 4
Source File: SortPartitionOperator.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {

		String name = "Sort at " + sortLocationName;

		if (useKeySelector) {
			return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
		}

		// flatten sort key positions
		List<Integer> allKeyPositions = new ArrayList<>();
		List<Order> allOrders = new ArrayList<>();
		for (int i = 0, length = keys.size(); i < length; i++) {
			int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
			Order order = orders.get(i);

			for (int sortKeyPosition : sortKeyPositions) {
				allKeyPositions.add(sortKeyPosition);
				allOrders.add(order);
			}
		}

		Ordering partitionOrdering = new Ordering();
		for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
			partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
		}

		// distinguish between partition types
		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
		SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
		noop.setInput(input);
		if (this.getParallelism() < 0) {
			// use parallelism of input if not explicitly specified
			noop.setParallelism(input.getParallelism());
		} else {
			// use explicitly specified parallelism
			noop.setParallelism(this.getParallelism());
		}

		return noop;

	}
Example 5
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {

		String name = "Sort at " + sortLocationName;

		if (useKeySelector) {
			return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
		}

		// flatten sort key positions
		List<Integer> allKeyPositions = new ArrayList<>();
		List<Order> allOrders = new ArrayList<>();
		for (int i = 0, length = keys.size(); i < length; i++) {
			int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
			Order order = orders.get(i);

			for (int sortKeyPosition : sortKeyPositions) {
				allKeyPositions.add(sortKeyPosition);
				allOrders.add(order);
			}
		}

		Ordering partitionOrdering = new Ordering();
		for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
			partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
		}

		// distinguish between partition types
		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
		SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
		noop.setInput(input);
		if (this.getParallelism() < 0) {
			// use parallelism of input if not explicitly specified
			noop.setParallelism(input.getParallelism());
		} else {
			// use explicitly specified parallelism
			noop.setParallelism(this.getParallelism());
		}

		return noop;

	}
Example 6
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {

		String name = "Sort at " + sortLocationName;

		if (useKeySelector) {
			return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
		}

		// flatten sort key positions
		List<Integer> allKeyPositions = new ArrayList<>();
		List<Order> allOrders = new ArrayList<>();
		for (int i = 0, length = keys.size(); i < length; i++) {
			int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
			Order order = orders.get(i);

			for (int sortKeyPosition : sortKeyPositions) {
				allKeyPositions.add(sortKeyPosition);
				allOrders.add(order);
			}
		}

		Ordering partitionOrdering = new Ordering();
		for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
			partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
		}

		// distinguish between partition types
		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
		SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
		noop.setInput(input);
		if (this.getParallelism() < 0) {
			// use parallelism of input if not explicitly specified
			noop.setParallelism(input.getParallelism());
		} else {
			// use explicitly specified parallelism
			noop.setParallelism(this.getParallelism());
		}

		return noop;

	}