org.apache.flink.api.common.ExecutionMode Java Examples

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

/**
 * This function connects the operators that produce the broadcast inputs to this operator.
 *
 * @param operatorToNode The map from program operators to optimizer nodes.
 * @param defaultExchangeMode The data exchange mode to use, if the operator does not
 *                            specify one.
 *
 * @throws CompilerException
 */
public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) {
	// skip for Operators that don't support broadcast variables 
	if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) {
		return;
	}

	// get all broadcast inputs
	AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator());

	// create connections and add them
	for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) {
		OptimizerNode predecessor = operatorToNode.get(input.getValue());
		DagConnection connection = new DagConnection(predecessor, this,
														ShipStrategyType.BROADCAST, defaultExchangeMode);
		addBroadcastConnection(input.getKey(), connection);
		predecessor.addOutgoingConnection(connection);
	}
}
 

private static JobGraph createJobGraph(
		ScheduleMode scheduleMode,
		ResultPartitionType resultPartitionType,
		ExecutionMode executionMode) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	JobVertex source = new JobVertex("source");
	source.setInvokableClass(LongValueSource.class);
	source.setParallelism(PARALLELISM);
	source.setSlotSharingGroup(slotSharingGroup);

	JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(ResultVerifyingSink.class);
	sink.setParallelism(PARALLELISM);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
	JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(scheduleMode);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setExecutionMode(executionMode);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

public SinkJoiner(OptimizerNode input1, OptimizerNode input2) {
	super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));

	DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED);
	DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED);
	
	this.input1 = conn1;
	this.input2 = conn2;
	
	setParallelism(1);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

@Test
public void testPipelineBreakerBroadcastedAllReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> bcInput1 = sourceWithMapper
									.map(new IdentityMapper<Long>())
									.reduce(new SelectOneReducer<Long>());
		DataSet<Long> bcInput2 = env.generateSequence(1, 10);
		
		DataSet<Long> result = sourceWithMapper
				.map(new IdentityMapper<Long>())
						.withBroadcastSet(bcInput1, "bc1")
						.withBroadcastSet(bcInput2, "bc2");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifyBranchingJoiningPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

/** These defaults should only be changed with a very good reason. */
@Test
public void testDefaults() {
  FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class);
  assertThat(options.getParallelism(), is(-1));
  assertThat(options.getMaxParallelism(), is(-1));
  assertThat(options.getFlinkMaster(), is("[auto]"));
  assertThat(options.getFilesToStage(), is(nullValue()));
  assertThat(options.getLatencyTrackingInterval(), is(0L));
  assertThat(options.getShutdownSourcesAfterIdleMs(), is(-1L));
  assertThat(options.getObjectReuse(), is(false));
  assertThat(options.getCheckpointingMode(), is(CheckpointingMode.EXACTLY_ONCE.name()));
  assertThat(options.getMinPauseBetweenCheckpoints(), is(-1L));
  assertThat(options.getCheckpointingInterval(), is(-1L));
  assertThat(options.getCheckpointTimeoutMillis(), is(-1L));
  assertThat(options.getNumConcurrentCheckpoints(), is(1));
  assertThat(options.getFailOnCheckpointingErrors(), is(true));
  assertThat(options.getFinishBundleBeforeCheckpointing(), is(false));
  assertThat(options.getNumberOfExecutionRetries(), is(-1));
  assertThat(options.getExecutionRetryDelay(), is(-1L));
  assertThat(options.getRetainExternalizedCheckpointsOnCancellation(), is(false));
  assertThat(options.getStateBackendFactory(), is(nullValue()));
  assertThat(options.getMaxBundleSize(), is(1000L));
  assertThat(options.getMaxBundleTimeMills(), is(1000L));
  assertThat(options.getExecutionModeForBatch(), is(ExecutionMode.PIPELINED.name()));
  assertThat(options.getSavepointPath(), is(nullValue()));
  assertThat(options.getAllowNonRestoredState(), is(false));
  assertThat(options.getDisableMetrics(), is(false));
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
	Operator<?> children = getOperator().getInput();

	final OptimizerNode pred;
	final DagConnection conn;
	
	pred = contractToNode.get(children);
	conn = new DagConnection(pred, this, defaultExchangeMode);
		
	// create the connection and add it
	this.input = conn;
	pred.addOutgoingConnection(conn);
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchigPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
	Operator<?> children = getOperator().getInput();

	final OptimizerNode pred;
	final DagConnection conn;
	
	pred = contractToNode.get(children);
	conn = new DagConnection(pred, this, defaultExchangeMode);
		
	// create the connection and add it
	this.input = conn;
	pred.addOutgoingConnection(conn);
}
 

public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
 

@Test
public void testPipelineBreakerWithBroadcastVariable() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> result = source.map(new IdentityMapper<Long>())
									.map(new IdentityMapper<Long>())
										.withBroadcastSet(source, "bc");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode mapperInput = (SingleInputPlanNode) mapper.getInput().getSource();
		
		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode());
		
		assertEquals(DataExchangeMode.BATCH, mapperInput.getInput().getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
		
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Gets the data exchange mode to use for this connection.
 *
 * @return The data exchange mode to use for this connection.
 */
public ExecutionMode getDataExchangeMode() {
	if (dataExchangeMode == null) {
		throw new IllegalStateException("This connection does not have the data exchange mode set");
	}
	return dataExchangeMode;
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchigPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchigPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}