org.apache.flink.runtime.operators.FlatMapDriver Java Examples

The following examples show how to use org.apache.flink.runtime.operators.FlatMapDriver. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ChainTaskTest.java From flink with Apache License 2.0

5 votes

@Test
public void testBatchTaskOutputInCloseMethod() {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	try {
		initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
		addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
		addOutput(outList);
		registerTask(FlatMapDriver.class, MockMapStub.class);
		for (int i = 0; i < numChainedTasks; i++) {
			final TaskConfig taskConfig = new TaskConfig(new Configuration());
			taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
			taskConfig.setOutputSerializer(serFact);
			taskConfig.setStubWrapper(
				new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
			getTaskConfig().addChainedTask(
				ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
		}
		final BatchTask<FlatMapFunction<Record, Record>, Record> testTask =
			new BatchTask<>(mockEnv);
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #2

Source File: ChainedAllReduceDriverTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example #3

Source File: ChainedAllReduceDriverTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example #4

Source File: ChainedAllReduceDriverTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}