Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#getParallelism()

The following examples show how to use org.apache.flink.api.java.ExecutionEnvironment#getParallelism() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PythonPlanBinder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void receiveParameters(ExecutionEnvironment env) throws IOException {
	for (int x = 0; x < Parameters.values().length; x++) {
		Tuple value = (Tuple) streamer.getRecord(true);
		switch (Parameters.valueOf(((String) value.getField(0)).toUpperCase())) {
			case DOP:
				Integer dop = value.<Integer>getField(1);
				env.setParallelism(dop);
				break;
			case RETRY:
				int retry = value.<Integer>getField(1);
				env.setRestartStrategy(RestartStrategies.fixedDelayRestart(retry, 10000L));
				break;
			case ID:
				currentEnvironmentID = value.<Integer>getField(1);
				break;
		}
	}
	if (env.getParallelism() < 0) {
		env.setParallelism(1);
	}
}
Example 2
Source File: PartitionITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testForcedRebalancing() throws Exception {
	/*
	 * Test forced rebalancing
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// generate some number in parallel
	DataSet<Long> ds = env.generateSequence(1, 3000);
	DataSet<Tuple2<Integer, Integer>> uniqLongs = ds
			// introduce some partition skew by filtering
			.filter(new Filter1())
			// rebalance
			.rebalance()
			// count values in each partition
			.map(new PartitionIndexMapper())
			.groupBy(0)
			.reduce(new Reducer1())
			// round counts to mitigate runtime scheduling effects (lazy split assignment)
			.map(new Mapper1());

	List<Tuple2<Integer, Integer>> result = uniqLongs.collect();

	StringBuilder expected = new StringBuilder();
	int numPerPartition = 2220 / env.getParallelism() / 10;
	for (int i = 0; i < env.getParallelism(); i++) {
		expected.append('(').append(i).append(',')
		.append(numPerPartition).append(")\n");
	}

	compareResultAsText(result, expected.toString());
}
Example 3
Source File: StreamPlanEnvironment.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected StreamPlanEnvironment(ExecutionEnvironment env) {
	super();
	this.env = env;

	int parallelism = env.getParallelism();
	if (parallelism > 0) {
		setParallelism(parallelism);
	} else {
		// determine parallelism
		setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM));
	}
}
Example 4
Source File: PartitionITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testForcedRebalancing() throws Exception {
	/*
	 * Test forced rebalancing
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// generate some number in parallel
	DataSet<Long> ds = env.generateSequence(1, 3000);
	DataSet<Tuple2<Integer, Integer>> uniqLongs = ds
			// introduce some partition skew by filtering
			.filter(new Filter1())
			// rebalance
			.rebalance()
			// count values in each partition
			.map(new PartitionIndexMapper())
			.groupBy(0)
			.reduce(new Reducer1())
			// round counts to mitigate runtime scheduling effects (lazy split assignment)
			.map(new Mapper1());

	List<Tuple2<Integer, Integer>> result = uniqLongs.collect();

	StringBuilder expected = new StringBuilder();
	int numPerPartition = 2220 / env.getParallelism() / 10;
	for (int i = 0; i < env.getParallelism(); i++) {
		expected.append('(').append(i).append(',')
		.append(numPerPartition).append(")\n");
	}

	compareResultAsText(result, expected.toString());
}
Example 5
Source File: StreamPlanEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected StreamPlanEnvironment(ExecutionEnvironment env) {
	super();
	this.env = env;

	int parallelism = env.getParallelism();
	if (parallelism > 0) {
		setParallelism(parallelism);
	} else {
		// determine parallelism
		setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM));
	}
}
Example 6
Source File: PartitionITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testForcedRebalancing() throws Exception {
	/*
	 * Test forced rebalancing
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// generate some number in parallel
	DataSet<Long> ds = env.generateSequence(1, 3000);
	DataSet<Tuple2<Integer, Integer>> uniqLongs = ds
			// introduce some partition skew by filtering
			.filter(new Filter1())
			// rebalance
			.rebalance()
			// count values in each partition
			.map(new PartitionIndexMapper())
			.groupBy(0)
			.reduce(new Reducer1())
			// round counts to mitigate runtime scheduling effects (lazy split assignment)
			.map(new Mapper1());

	List<Tuple2<Integer, Integer>> result = uniqLongs.collect();

	StringBuilder expected = new StringBuilder();
	int numPerPartition = 2220 / env.getParallelism() / 10;
	for (int i = 0; i < env.getParallelism(); i++) {
		expected.append('(').append(i).append(',')
		.append(numPerPartition).append(")\n");
	}

	compareResultAsText(result, expected.toString());
}