org.apache.flink.api.common.functions.util.RuntimeUDFContext Java Examples

private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
		throws Exception {
	@SuppressWarnings("unchecked")
	GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
	
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}
	return typedSource.executeOnCollections(ctx, executionConfig);
}
 

private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
		throws Exception {
	@SuppressWarnings("unchecked")
	GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
	
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}
	return typedSource.executeOnCollections(ctx, executionConfig);
}
 

private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 

private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 

private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 

private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
		throws Exception {
	@SuppressWarnings("unchecked")
	GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
	
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}
	return typedSource.executeOnCollections(ctx, executionConfig);
}
 

private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp = operator.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> inputData = (List<IN>) execute(inputOp, superStep);
	
	@SuppressWarnings("unchecked")
	SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedInputFormat<Value> inputFormat = new SerializedInputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	inputFormat.setRuntimeContext(
			new RuntimeUDFContext(
					taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
					new HashMap<String, Future<Path>>(),
					new HashMap<String, Accumulator<?, ?>>(),
					new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedInputFormat<Value> inputFormat = new SerializedInputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	inputFormat.setRuntimeContext(
			new RuntimeUDFContext(
					taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
					new HashMap<String, Future<Path>>(),
					new HashMap<String, Accumulator<?, ?>>(),
					new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 

private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp = operator.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> inputData = (List<IN>) execute(inputOp, superStep);
	
	@SuppressWarnings("unchecked")
	SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}
 

private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 

@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedInputFormat<Value> inputFormat = new SerializedInputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	inputFormat.setRuntimeContext(
			new RuntimeUDFContext(
					taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
					new HashMap<String, Future<Path>>(),
					new HashMap<String, Accumulator<?, ?>>(),
					new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp = operator.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> inputData = (List<IN>) execute(inputOp, superStep);
	
	@SuppressWarnings("unchecked")
	SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}
 

@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 

@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testTupleBaseJoiner(){
	final FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>> joiner =
				new FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>>() {
		@Override
		public void join(Tuple3<String, Double, Integer> first, Tuple2<Integer, String> second, Collector<Tuple2<Double, String>> out) {

			assertEquals(first.f0, second.f1);
			assertEquals(first.f2, second.f0);

			out.collect(new Tuple2<>(first.f1, second.f0.toString()));
		}
	};

	final TupleTypeInfo<Tuple3<String, Double, Integer>> leftTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo
			(String.class, Double.class, Integer.class);
	final TupleTypeInfo<Tuple2<Integer, String>> rightTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Integer.class,
			String.class);
	final TupleTypeInfo<Tuple2<Double, String>> outTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Double.class,
			String.class);

	final int[] leftKeys = new int[]{0, 2};
	final int[] rightKeys = new int[]{1, 0};

	final String taskName = "Collection based tuple joiner";

	final BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double,
			String>> binaryOpInfo = new BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer,
			String>, Tuple2<Double, String>>(leftTypeInfo, rightTypeInfo, outTypeInfo);

	final InnerJoinOperatorBase<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>>> base = new InnerJoinOperatorBase<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>>>(joiner, binaryOpInfo, leftKeys, rightKeys, taskName);

	final List<Tuple3<String, Double, Integer> > inputData1 = new ArrayList<Tuple3<String, Double,
			Integer>>(Arrays.asList(
			new Tuple3<>("foo", 42.0, 1),
			new Tuple3<>("bar", 1.0, 2),
			new Tuple3<>("bar", 2.0, 3),
			new Tuple3<>("foobar", 3.0, 4),
			new Tuple3<>("bar", 3.0, 3)
	));

	final List<Tuple2<Integer, String>> inputData2 = new ArrayList<Tuple2<Integer, String>>(Arrays.asList(
			new Tuple2<>(3, "bar"),
			new Tuple2<>(4, "foobar"),
			new Tuple2<>(2, "foo")
	));
	final Set<Tuple2<Double, String>> expected = new HashSet<Tuple2<Double, String>>(Arrays.asList(
			new Tuple2<>(2.0, "3"),
			new Tuple2<>(3.0, "3"),
			new Tuple2<>(3.0, "4")
	));

	try {
		final TaskInfo taskInfo = new TaskInfo("op", 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, new HashSet<>(resultSafe));
		assertEquals(expected, new HashSet<>(resultRegular));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}