Java Code Examples for org.apache.flink.runtime.jobgraph.JobGraph#addJar()

The following examples show how to use org.apache.flink.runtime.jobgraph.JobGraph#addJar() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
Example 2
Source File: ClusterClient.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) {
	JobGraph job;
	if (optPlan instanceof StreamingPlan) {
		job = ((StreamingPlan) optPlan).getJobGraph();
		job.setSavepointRestoreSettings(savepointSettings);
	} else {
		JobGraphGenerator gen = new JobGraphGenerator(flinkConfig);
		job = gen.compileJobGraph((OptimizedPlan) optPlan);
	}

	for (URL jar : jarFiles) {
		try {
			job.addJar(new Path(jar.toURI()));
		} catch (URISyntaxException e) {
			throw new RuntimeException("URL is invalid. This should not happen.", e);
		}
	}

	job.setClasspaths(classpaths);

	return job;
}
Example 3
Source File: TestEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
Example 4
Source File: ClusterClient.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) {
	JobGraph job;
	if (optPlan instanceof StreamingPlan) {
		job = ((StreamingPlan) optPlan).getJobGraph();
		job.setSavepointRestoreSettings(savepointSettings);
	} else {
		JobGraphGenerator gen = new JobGraphGenerator(flinkConfig);
		job = gen.compileJobGraph((OptimizedPlan) optPlan);
	}

	for (URL jar : jarFiles) {
		try {
			job.addJar(new Path(jar.toURI()));
		} catch (URISyntaxException e) {
			throw new RuntimeException("URL is invalid. This should not happen.", e);
		}
	}

	job.setClasspaths(classpaths);

	return job;
}
Example 5
Source File: DFCusterClient.java    From df_data_service with Apache License 2.0 6 votes vote down vote up 
private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) {
	JobGraph job;
	if (optPlan instanceof StreamingPlan) {
		job = ((StreamingPlan) optPlan).getJobGraph();
		job.setSavepointRestoreSettings(savepointSettings);
	} else {
		JobGraphGenerator gen = new JobGraphGenerator(this.flinkConfig);
		job = gen.compileJobGraph((OptimizedPlan) optPlan);
	}

	for (URL jar : jarFiles) {
		try {
			job.addJar(new Path(jar.toURI()));
		} catch (URISyntaxException e) {
			throw new RuntimeException("URL is invalid. This should not happen.", e);
		}
	}

	job.setClasspaths(classpaths);

	return job;
}
Example 6
Source File: TestEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
Example 7
Source File: TestStreamEnvironment.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	final StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);
	final JobGraph jobGraph = streamGraph.getJobGraph();

	for (Path jarFile : jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	return jobExecutor.executeJobBlocking(jobGraph);
}
Example 8
Source File: TestStreamEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	final JobGraph jobGraph = streamGraph.getJobGraph();
	transformations.clear();

	for (Path jarFile : jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	return jobExecutor.executeJobBlocking(jobGraph);
}
Example 9
Source File: TestStreamEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	final JobGraph jobGraph = streamGraph.getJobGraph();

	for (Path jarFile : jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	return jobExecutor.executeJobBlocking(jobGraph);
}
Example 10
Source File: YARNFileReplicationITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void deployPerJob(Configuration configuration, JobGraph jobGraph) throws Exception {
	try (final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor(configuration)) {

		yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
		yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));

		final int masterMemory = yarnClusterDescriptor.getFlinkConfiguration().get(JobManagerOptions.TOTAL_PROCESS_MEMORY).getMebiBytes();
		final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(masterMemory)
			.setTaskManagerMemoryMB(1024)
			.setSlotsPerTaskManager(1)
			.createClusterSpecification();

		File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));

		jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
		try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
			.deployJobCluster(
				clusterSpecification,
				jobGraph,
				false)
			.getClusterClient()) {

			ApplicationId applicationId = clusterClient.getClusterId();

			final CompletableFuture<JobResult> jobResultCompletableFuture = clusterClient.requestJobResult(jobGraph.getJobID());

			final JobResult jobResult = jobResultCompletableFuture.get();

			assertThat(jobResult, is(notNullValue()));
			assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));

			extraVerification(configuration, applicationId);

			waitApplicationFinishedElseKillIt(
				applicationId, yarnAppTerminateTimeout, yarnClusterDescriptor, sleepIntervalInMS);
		}
	}
}
Example 11
Source File: PackagedProgramUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Creates a {@link JobGraph} with a specified {@link JobID}
 * from the given {@link PackagedProgram}.
 *
 * @param packagedProgram to extract the JobGraph from
 * @param configuration to use for the optimizer and job graph generator
 * @param defaultParallelism for the JobGraph
 * @param jobID the pre-generated job id
 * @return JobGraph extracted from the PackagedProgram
 * @throws ProgramInvocationException if the JobGraph generation failed
 */
public static JobGraph createJobGraph(
		PackagedProgram packagedProgram,
		Configuration configuration,
		int defaultParallelism,
		@Nullable JobID jobID) throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader());
	final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
	final FlinkPlan flinkPlan;

	if (packagedProgram.isUsingProgramEntryPoint()) {

		final JobWithJars jobWithJars = packagedProgram.getPlanWithJars();

		final Plan plan = jobWithJars.getPlan();

		if (plan.getDefaultParallelism() <= 0) {
			plan.setDefaultParallelism(defaultParallelism);
		}

		flinkPlan = optimizer.compile(jobWithJars.getPlan());
	} else if (packagedProgram.isUsingInteractiveMode()) {
		final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);

		optimizerPlanEnvironment.setParallelism(defaultParallelism);

		flinkPlan = optimizerPlanEnvironment.getOptimizedPlan(packagedProgram);
	} else {
		throw new ProgramInvocationException("PackagedProgram does not have a valid invocation mode.");
	}

	final JobGraph jobGraph;

	if (flinkPlan instanceof StreamingPlan) {
		jobGraph = ((StreamingPlan) flinkPlan).getJobGraph(jobID);
		jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings());
	} else {
		final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration);
		jobGraph = jobGraphGenerator.compileJobGraph((OptimizedPlan) flinkPlan, jobID);
	}

	for (URL url : packagedProgram.getAllLibraries()) {
		try {
			jobGraph.addJar(new Path(url.toURI()));
		} catch (URISyntaxException e) {
			throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', jobGraph.getJobID(), e);
		}
	}

	jobGraph.setClasspaths(packagedProgram.getClasspaths());

	return jobGraph;
}
Example 12
Source File: YARNITCase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPerJobMode() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s");
	final YarnClient yarnClient = getYarnClient();

	try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
		configuration,
		getYarnConfiguration(),
		System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
		yarnClient,
		true)) {

		yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
		yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
		yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));

		final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(768)
			.setTaskManagerMemoryMB(1024)
			.setSlotsPerTaskManager(1)
			.setNumberTaskManagers(1)
			.createClusterSpecification();

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(2);

		env.addSource(new NoDataSource())
			.shuffle()
			.addSink(new DiscardingSink<>());

		final JobGraph jobGraph = env.getStreamGraph().getJobGraph();

		File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));

		jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));

		ApplicationId applicationId = null;
		ClusterClient<ApplicationId> clusterClient = null;

		try {
			clusterClient = yarnClusterDescriptor.deployJobCluster(
				clusterSpecification,
				jobGraph,
				false);
			applicationId = clusterClient.getClusterId();

			assertThat(clusterClient, is(instanceOf(RestClusterClient.class)));
			final RestClusterClient<ApplicationId> restClusterClient = (RestClusterClient<ApplicationId>) clusterClient;

			final CompletableFuture<JobResult> jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID());

			final JobResult jobResult = jobResultCompletableFuture.get();

			assertThat(jobResult, is(notNullValue()));
			assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
		} finally {
			if (clusterClient != null) {
				clusterClient.shutdown();
			}

			if (applicationId != null) {
				yarnClusterDescriptor.killCluster(applicationId);
			}
		}
	}
}
Example 13
Source File: PackagedProgramUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Creates a {@link JobGraph} with a specified {@link JobID}
 * from the given {@link PackagedProgram}.
 *
 * @param packagedProgram to extract the JobGraph from
 * @param configuration to use for the optimizer and job graph generator
 * @param defaultParallelism for the JobGraph
 * @param jobID the pre-generated job id
 * @return JobGraph extracted from the PackagedProgram
 * @throws ProgramInvocationException if the JobGraph generation failed
 */
public static JobGraph createJobGraph(
		PackagedProgram packagedProgram,
		Configuration configuration,
		int defaultParallelism,
		@Nullable JobID jobID) throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader());
	final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
	final FlinkPlan flinkPlan;

	if (packagedProgram.isUsingProgramEntryPoint()) {

		final JobWithJars jobWithJars = packagedProgram.getPlanWithJars();

		final Plan plan = jobWithJars.getPlan();

		if (plan.getDefaultParallelism() <= 0) {
			plan.setDefaultParallelism(defaultParallelism);
		}

		flinkPlan = optimizer.compile(jobWithJars.getPlan());
	} else if (packagedProgram.isUsingInteractiveMode()) {
		final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);

		optimizerPlanEnvironment.setParallelism(defaultParallelism);

		flinkPlan = optimizerPlanEnvironment.getOptimizedPlan(packagedProgram);
	} else {
		throw new ProgramInvocationException("PackagedProgram does not have a valid invocation mode.");
	}

	final JobGraph jobGraph;

	if (flinkPlan instanceof StreamingPlan) {
		jobGraph = ((StreamingPlan) flinkPlan).getJobGraph(jobID);
		jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings());
	} else {
		final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration);
		jobGraph = jobGraphGenerator.compileJobGraph((OptimizedPlan) flinkPlan, jobID);
	}

	for (URL url : packagedProgram.getAllLibraries()) {
		try {
			jobGraph.addJar(new Path(url.toURI()));
		} catch (URISyntaxException e) {
			throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', jobGraph.getJobID(), e);
		}
	}

	jobGraph.setClasspaths(packagedProgram.getClasspaths());

	return jobGraph;
}
Example 14
Source File: YARNITCase.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void deployPerJob(Configuration configuration, JobGraph jobGraph, boolean withDist) throws Exception {
	try (final YarnClusterDescriptor yarnClusterDescriptor = withDist
			? createYarnClusterDescriptor(configuration)
			: createYarnClusterDescriptorWithoutLibDir(configuration)) {

		final int masterMemory = yarnClusterDescriptor.getFlinkConfiguration().get(JobManagerOptions.TOTAL_PROCESS_MEMORY).getMebiBytes();
		final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(masterMemory)
			.setTaskManagerMemoryMB(1024)
			.setSlotsPerTaskManager(1)
			.createClusterSpecification();

		File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));

		jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
		try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
				.deployJobCluster(
						clusterSpecification,
						jobGraph,
						false)
				.getClusterClient()) {

			for (DistributedCache.DistributedCacheEntry entry : jobGraph.getUserArtifacts().values()) {
				assertTrue(
					String.format("The user artifacts(%s) should be remote or uploaded to remote filesystem.", entry.filePath),
					Utils.isRemotePath(entry.filePath));
			}

			ApplicationId applicationId = clusterClient.getClusterId();

			final CompletableFuture<JobResult> jobResultCompletableFuture = clusterClient.requestJobResult(jobGraph.getJobID());

			final JobResult jobResult = jobResultCompletableFuture.get();

			assertThat(jobResult, is(notNullValue()));
			assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));

			checkStagingDirectory(configuration, applicationId);

			waitApplicationFinishedElseKillIt(
				applicationId, yarnAppTerminateTimeout, yarnClusterDescriptor, sleepIntervalInMS);
		}
	}
}