Java Code Examples for org.apache.flink.api.common.ExecutionConfig#enableObjectReuse()

@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {

	LinkedList<Object[]> configs = new LinkedList<Object[]>();

	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	configs.add(a);
	Object[] b = { withReuse };
	configs.add(b);

	return configs;
}
 

private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 

@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 

@Test
public void testDataSourcePlain() {
	try {
		TestNonRichInputFormat in = new TestNonRichInputFormat();
		GenericDataSourceBase<String, TestNonRichInputFormat> source =
				new GenericDataSourceBase<String, TestNonRichInputFormat>(
						in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig);

		in.reset();
		executionConfig.enableObjectReuse();
		List<String> resultRegular = source.executeOnCollections(null, executionConfig);
		assertEquals(asList(TestIOData.NAMES), resultMutableSafe);
		assertEquals(asList(TestIOData.NAMES), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 

@Test
public void testDataSourcePlain() {
	try {
		TestNonRichOutputFormat out = new TestNonRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	Object[] b = { withReuse };
	return Arrays.asList(a, b);
}
 

@Test
public void testMapPlain() {
	try {
		final MapFunction<String, Integer> parser = new MapFunction<String, Integer>() {
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), "TestMapper");
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Integer> resultMutableSafe = op.executeOnCollections(input, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input, null, executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	Object[] b = { withReuse };
	return Arrays.asList(a, b);
}
 

@Test
public void testReduceCollection() {
	try {
		final ReduceFunction<Tuple2<String, Integer>> reducer =
				(value1, value2) -> new Tuple2<>(value1.f0, value1.f1 + value2.f1);

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, null, executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 

@Test
public void testMapWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapFunction<String, Integer> parser = new RichMapFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 

@Test
public void testMapWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapFunction<String, Integer> parser = new RichMapFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	JobGraph jobGraph = new JobGraph(v1, v2);
	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	jobGraph.setExecutionConfig(config);

	runtimeGraph = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(jobGraph)
		.build();

	runtimeGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		false,
		0);

	runtimeGraph.enableCheckpointing(
		chkConfig,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}
 

@Test
public void testMapPartitionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapPartitionFunction<String, Integer> parser = new RichMapPartitionFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public void mapPartition(Iterable<String> values, Collector<Integer> out) {
				for (String s : values) {
					out.collect(Integer.parseInt(s));
				}
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String, Integer>> op = 
				new MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	List<JobVertex> vertices = new ArrayList<>(Arrays.asList(v1, v2));

	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	runtimeGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"test job",
		new Configuration(),
		new SerializedValue<>(config),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		mock(SlotProvider.class));

	runtimeGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	runtimeGraph.attachJobGraph(vertices);

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);

	runtimeGraph.enableCheckpointing(
		chkConfig,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}
 

@Test
public void testTupleBaseJoiner(){
	final FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>> joiner =
				new FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>>() {
		@Override
		public void join(Tuple3<String, Double, Integer> first, Tuple2<Integer, String> second, Collector<Tuple2<Double, String>> out) {

			assertEquals(first.f0, second.f1);
			assertEquals(first.f2, second.f0);

			out.collect(new Tuple2<>(first.f1, second.f0.toString()));
		}
	};

	final TupleTypeInfo<Tuple3<String, Double, Integer>> leftTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo
			(String.class, Double.class, Integer.class);
	final TupleTypeInfo<Tuple2<Integer, String>> rightTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Integer.class,
			String.class);
	final TupleTypeInfo<Tuple2<Double, String>> outTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Double.class,
			String.class);

	final int[] leftKeys = new int[]{0, 2};
	final int[] rightKeys = new int[]{1, 0};

	final String taskName = "Collection based tuple joiner";

	final BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double,
			String>> binaryOpInfo = new BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer,
			String>, Tuple2<Double, String>>(leftTypeInfo, rightTypeInfo, outTypeInfo);

	final InnerJoinOperatorBase<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>>> base = new InnerJoinOperatorBase<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>>>(joiner, binaryOpInfo, leftKeys, rightKeys, taskName);

	final List<Tuple3<String, Double, Integer> > inputData1 = new ArrayList<Tuple3<String, Double,
			Integer>>(Arrays.asList(
			new Tuple3<>("foo", 42.0, 1),
			new Tuple3<>("bar", 1.0, 2),
			new Tuple3<>("bar", 2.0, 3),
			new Tuple3<>("foobar", 3.0, 4),
			new Tuple3<>("bar", 3.0, 3)
	));

	final List<Tuple2<Integer, String>> inputData2 = new ArrayList<Tuple2<Integer, String>>(Arrays.asList(
			new Tuple2<>(3, "bar"),
			new Tuple2<>(4, "foobar"),
			new Tuple2<>(2, "foo")
	));
	final Set<Tuple2<Double, String>> expected = new HashSet<Tuple2<Double, String>>(Arrays.asList(
			new Tuple2<>(2.0, "3"),
			new Tuple2<>(3.0, "3"),
			new Tuple2<>(3.0, "4")
	));

	try {
		final TaskInfo taskInfo = new TaskInfo("op", 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, new HashSet<>(resultSafe));
		assertEquals(expected, new HashSet<>(resultRegular));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}