Java Code Examples for org.apache.flink.runtime.jobgraph.JobVertex#setMaxParallelism()

The following examples show how to use org.apache.flink.runtime.jobgraph.JobVertex#setMaxParallelism() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ExecutionJobVertexTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}
Example 2
Source File: ExecutionGraphRescalingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static JobVertex[] createVerticesForSimpleBipartiteJobGraph(int parallelism, int maxParallelism) {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");

	JobVertex[] jobVertices = new JobVertex[]{v1, v2, v3, v4, v5};

	for (JobVertex jobVertex : jobVertices) {
		jobVertex.setInvokableClass(AbstractInvokable.class);
		jobVertex.setParallelism(parallelism);
		jobVertex.setMaxParallelism(maxParallelism);
	}

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	return jobVertices;
}
Example 3
Source File: ExecutionJobVertexTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}
Example 4
Source File: ExecutionGraphRescalingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static JobVertex[] createVerticesForSimpleBipartiteJobGraph(int parallelism, int maxParallelism) {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");

	JobVertex[] jobVertices = new JobVertex[]{v1, v2, v3, v4, v5};

	for (JobVertex jobVertex : jobVertices) {
		jobVertex.setInvokableClass(AbstractInvokable.class);
		jobVertex.setParallelism(parallelism);
		jobVertex.setMaxParallelism(maxParallelism);
	}

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	return jobVertices;
}
Example 5
Source File: ExecutionGraphRescalingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static JobVertex[] createVerticesForSimpleBipartiteJobGraph(int parallelism, int maxParallelism) {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");

	JobVertex[] jobVertices = new JobVertex[]{v1, v2, v3, v4, v5};

	for (JobVertex jobVertex : jobVertices) {
		jobVertex.setInvokableClass(AbstractInvokable.class);
		jobVertex.setParallelism(parallelism);
		jobVertex.setMaxParallelism(maxParallelism);
	}

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	return jobVertices;
}
Example 6
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Rescales the given operators of the {@link JobGraph} of this {@link JobMaster} with respect to given
 * parallelism and {@link RescalingBehaviour}.
 *
 * @param operators to rescale
 * @param newParallelism new parallelism for these operators
 * @param rescalingBehaviour of the rescaling operation
 * @throws FlinkException if the {@link JobGraph} could not be rescaled
 */
private void rescaleJobGraph(Collection<JobVertexID> operators, int newParallelism, RescalingBehaviour rescalingBehaviour) throws FlinkException {
	for (JobVertexID jobVertexId : operators) {
		final JobVertex jobVertex = jobGraph.findVertexByID(jobVertexId);

		// update max parallelism in case that it has not been configured
		final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

		if (executionJobVertex != null) {
			jobVertex.setMaxParallelism(executionJobVertex.getMaxParallelism());
		}

		rescalingBehaviour.accept(jobVertex, newParallelism);
	}
}
Example 7
Source File: MiniClusterITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
Example 8
Source File: MiniClusterITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private static JobGraph getSimpleJob(int parallelism) throws IOException {
	final JobVertex task = new JobVertex("Test task");
	task.setParallelism(parallelism);
	task.setMaxParallelism(parallelism);
	task.setInvokableClass(NoOpInvokable.class);

	final JobGraph jg = new JobGraph(new JobID(), "Test Job", task);
	jg.setScheduleMode(ScheduleMode.EAGER);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000));
	jg.setExecutionConfig(executionConfig);

	return jg;
}
Example 9
Source File: JobMasterTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
Example 10
Source File: MiniClusterITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
Example 11
Source File: MiniClusterITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static JobGraph getSimpleJob(int parallelism) throws IOException {
	final JobVertex task = new JobVertex("Test task");
	task.setParallelism(parallelism);
	task.setMaxParallelism(parallelism);
	task.setInvokableClass(NoOpInvokable.class);

	final JobGraph jg = new JobGraph(new JobID(), "Test Job", task);
	jg.setScheduleMode(ScheduleMode.EAGER);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000));
	jg.setExecutionConfig(executionConfig);

	return jg;
}
Example 12
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
Example 13
Source File: MiniClusterITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
Example 14
Source File: MiniClusterITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static JobGraph getSimpleJob(int parallelism) throws IOException {
	final JobVertex task = new JobVertex("Test task");
	task.setParallelism(parallelism);
	task.setMaxParallelism(parallelism);
	task.setInvokableClass(NoOpInvokable.class);

	final JobGraph jg = new JobGraph(new JobID(), "Test Job", task);
	jg.setScheduleMode(ScheduleMode.EAGER);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000));
	jg.setExecutionConfig(executionConfig);

	return jg;
}
Example 15
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
Example 16
Source File: ExecutionJobVertexTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException, JobExecutionException {
	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraph = createExecutionGraph();
	return new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(10));
}