Java Code Examples for org.apache.flink.api.common.Plan#setDefaultParallelism()

private JobGraph getJobGraph(Pipeline pipeline, Configuration configuration) throws MalformedURLException {
	// This is a quirk in how LocalEnvironment used to work. It sets the default parallelism
	// to <num taskmanagers> * <num task slots>. Might be questionable but we keep the behaviour
	// for now.
	if (pipeline instanceof Plan) {
		Plan plan = (Plan) pipeline;
		final int slotsPerTaskManager = configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, plan.getMaximumParallelism());
		final int numTaskManagers = configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);

		plan.setDefaultParallelism(slotsPerTaskManager * numTaskManagers);
	}

	return PipelineExecutorUtils.getJobGraph(pipeline, configuration);
}
 

public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException {
	Logger log = LoggerFactory.getLogger(ClusterClient.class);

	if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
		log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism);
		p.setDefaultParallelism(parallelism);
	}
	log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism());

	return compiler.compile(p);
}
 

public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException {
	Logger log = LoggerFactory.getLogger(ClusterClient.class);

	if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
		log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism);
		p.setDefaultParallelism(parallelism);
	}
	log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism());

	return compiler.compile(p);
}
 

/**
 * Create plan.
 *
 * @return the generated plan.
 */
private Plan createPlan() {
	final OperatorTranslation translator = new OperatorTranslation();
	final Plan plan = translator.translateToPlan(sinks, jobName);

	if (defaultParallelism > 0) {
		plan.setDefaultParallelism(defaultParallelism);
	}
	plan.setExecutionConfig(config);
	return plan;
}
 

private void setDefaultParallelism(Plan plan, int defaultParallelism) {
	if (defaultParallelism > 0 && plan.getDefaultParallelism() <= 0) {
		LOG.debug(
				"Changing plan default parallelism from {} to {}",
				plan.getDefaultParallelism(),
				defaultParallelism);
		plan.setDefaultParallelism(defaultParallelism);
	}

	LOG.debug(
			"Set parallelism {}, plan default parallelism {}",
			defaultParallelism,
			plan.getDefaultParallelism());
}
 

/**
 * Executes the given program on a local runtime and waits for the job to finish.
 *
 * <p>If the executor has not been started before, this starts the executor and shuts it down
 * after the job finished. If the job runs in session mode, the executor is kept alive until
 * no more references to the executor exist.</p>
 *
 * @param plan The plan of the program to execute.
 * @return The net runtime of the program, in milliseconds.
 *
 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
 *                   caused an exception.
 */
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
	if (plan == null) {
		throw new IllegalArgumentException("The plan may not be null.");
	}

	synchronized (this.lock) {

		// check if we start a session dedicated for this execution
		final boolean shutDownAtEnd;

		if (jobExecutorService == null) {
			shutDownAtEnd = true;

			// configure the number of local slots equal to the parallelism of the local plan
			if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
				int maxParallelism = plan.getMaximumParallelism();
				if (maxParallelism > 0) {
					this.taskManagerNumSlots = maxParallelism;
				}
			}

			// start the cluster for us
			start();
		}
		else {
			// we use the existing session
			shutDownAtEnd = false;
		}

		try {
			// TODO: Set job's default parallelism to max number of slots
			final int slotsPerTaskManager = jobExecutorServiceConfiguration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskManagerNumSlots);
			final int numTaskManagers = jobExecutorServiceConfiguration.getInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
			plan.setDefaultParallelism(slotsPerTaskManager * numTaskManagers);

			Optimizer pc = new Optimizer(new DataStatistics(), jobExecutorServiceConfiguration);
			OptimizedPlan op = pc.compile(plan);

			JobGraphGenerator jgg = new JobGraphGenerator(jobExecutorServiceConfiguration);
			JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());

			return jobExecutorService.executeJobBlocking(jobGraph);
		}
		finally {
			if (shutDownAtEnd) {
				stop();
			}
		}
	}
}
 

/**
 * Creates a {@link JobGraph} with a specified {@link JobID}
 * from the given {@link PackagedProgram}.
 *
 * @param packagedProgram to extract the JobGraph from
 * @param configuration to use for the optimizer and job graph generator
 * @param defaultParallelism for the JobGraph
 * @param jobID the pre-generated job id
 * @return JobGraph extracted from the PackagedProgram
 * @throws ProgramInvocationException if the JobGraph generation failed
 */
public static JobGraph createJobGraph(
		PackagedProgram packagedProgram,
		Configuration configuration,
		int defaultParallelism,
		@Nullable JobID jobID) throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader());
	final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
	final FlinkPlan flinkPlan;

	if (packagedProgram.isUsingProgramEntryPoint()) {

		final JobWithJars jobWithJars = packagedProgram.getPlanWithJars();

		final Plan plan = jobWithJars.getPlan();

		if (plan.getDefaultParallelism() <= 0) {
			plan.setDefaultParallelism(defaultParallelism);
		}

		flinkPlan = optimizer.compile(jobWithJars.getPlan());
	} else if (packagedProgram.isUsingInteractiveMode()) {
		final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);

		optimizerPlanEnvironment.setParallelism(defaultParallelism);

		flinkPlan = optimizerPlanEnvironment.getOptimizedPlan(packagedProgram);
	} else {
		throw new ProgramInvocationException("PackagedProgram does not have a valid invocation mode.");
	}

	final JobGraph jobGraph;

	if (flinkPlan instanceof StreamingPlan) {
		jobGraph = ((StreamingPlan) flinkPlan).getJobGraph(jobID);
		jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings());
	} else {
		final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration);
		jobGraph = jobGraphGenerator.compileJobGraph((OptimizedPlan) flinkPlan, jobID);
	}

	for (URL url : packagedProgram.getAllLibraries()) {
		try {
			jobGraph.addJar(new Path(url.toURI()));
		} catch (URISyntaxException e) {
			throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', jobGraph.getJobID(), e);
		}
	}

	jobGraph.setClasspaths(packagedProgram.getClasspaths());

	return jobGraph;
}
 

/**
 * Executes the given program on a local runtime and waits for the job to finish.
 *
 * <p>If the executor has not been started before, this starts the executor and shuts it down
 * after the job finished. If the job runs in session mode, the executor is kept alive until
 * no more references to the executor exist.</p>
 *
 * @param plan The plan of the program to execute.
 * @return The net runtime of the program, in milliseconds.
 *
 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
 *                   caused an exception.
 */
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
	if (plan == null) {
		throw new IllegalArgumentException("The plan may not be null.");
	}

	synchronized (this.lock) {

		// check if we start a session dedicated for this execution
		final boolean shutDownAtEnd;

		if (jobExecutorService == null) {
			shutDownAtEnd = true;

			// configure the number of local slots equal to the parallelism of the local plan
			if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
				int maxParallelism = plan.getMaximumParallelism();
				if (maxParallelism > 0) {
					this.taskManagerNumSlots = maxParallelism;
				}
			}

			// start the cluster for us
			start();
		}
		else {
			// we use the existing session
			shutDownAtEnd = false;
		}

		try {
			// TODO: Set job's default parallelism to max number of slots
			final int slotsPerTaskManager = jobExecutorServiceConfiguration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskManagerNumSlots);
			final int numTaskManagers = jobExecutorServiceConfiguration.getInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
			plan.setDefaultParallelism(slotsPerTaskManager * numTaskManagers);

			Optimizer pc = new Optimizer(new DataStatistics(), jobExecutorServiceConfiguration);
			OptimizedPlan op = pc.compile(plan);

			JobGraphGenerator jgg = new JobGraphGenerator(jobExecutorServiceConfiguration);
			JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());

			return jobExecutorService.executeJobBlocking(jobGraph);
		}
		finally {
			if (shutDownAtEnd) {
				stop();
			}
		}
	}
}
 

/**
 * Creates a {@link JobGraph} with a specified {@link JobID}
 * from the given {@link PackagedProgram}.
 *
 * @param packagedProgram to extract the JobGraph from
 * @param configuration to use for the optimizer and job graph generator
 * @param defaultParallelism for the JobGraph
 * @param jobID the pre-generated job id
 * @return JobGraph extracted from the PackagedProgram
 * @throws ProgramInvocationException if the JobGraph generation failed
 */
public static JobGraph createJobGraph(
		PackagedProgram packagedProgram,
		Configuration configuration,
		int defaultParallelism,
		@Nullable JobID jobID) throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader());
	final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
	final FlinkPlan flinkPlan;

	if (packagedProgram.isUsingProgramEntryPoint()) {

		final JobWithJars jobWithJars = packagedProgram.getPlanWithJars();

		final Plan plan = jobWithJars.getPlan();

		if (plan.getDefaultParallelism() <= 0) {
			plan.setDefaultParallelism(defaultParallelism);
		}

		flinkPlan = optimizer.compile(jobWithJars.getPlan());
	} else if (packagedProgram.isUsingInteractiveMode()) {
		final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);

		optimizerPlanEnvironment.setParallelism(defaultParallelism);

		flinkPlan = optimizerPlanEnvironment.getOptimizedPlan(packagedProgram);
	} else {
		throw new ProgramInvocationException("PackagedProgram does not have a valid invocation mode.");
	}

	final JobGraph jobGraph;

	if (flinkPlan instanceof StreamingPlan) {
		jobGraph = ((StreamingPlan) flinkPlan).getJobGraph(jobID);
		jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings());
	} else {
		final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration);
		jobGraph = jobGraphGenerator.compileJobGraph((OptimizedPlan) flinkPlan, jobID);
	}

	for (URL url : packagedProgram.getAllLibraries()) {
		try {
			jobGraph.addJar(new Path(url.toURI()));
		} catch (URISyntaxException e) {
			throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', jobGraph.getJobID(), e);
		}
	}

	jobGraph.setClasspaths(packagedProgram.getClasspaths());

	return jobGraph;
}