org.apache.flink.runtime.client.JobSubmissionException Java Examples

private CompletableFuture<Acknowledge> internalSubmitJob(JobGraph jobGraph) {
	log.info("Submitting job {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	final CompletableFuture<Acknowledge> persistAndRunFuture = waitForTerminatingJobManager(jobGraph.getJobID(), jobGraph, this::persistAndRunJob)
		.thenApply(ignored -> Acknowledge.get());

	return persistAndRunFuture.handleAsync((acknowledge, throwable) -> {
		if (throwable != null) {
			cleanUpJobData(jobGraph.getJobID(), true);

			final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
			log.error("Failed to submit job {}.", jobGraph.getJobID(), strippedThrowable);
			throw new CompletionException(
				new JobSubmissionException(jobGraph.getJobID(), "Failed to submit job.", strippedThrowable));
		} else {
			return acknowledge;
		}
	}, getRpcService().getExecutor());
}
 

/**
 * Tests that a duplicate job submission won't delete any job meta data
 * (submitted job graphs, blobs, etc.).
 */
@Test
public void testDuplicateJobSubmissionDoesNotDeleteJobMetaData() throws Exception {
	submitJob();

	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		try {
			submissionFuture.get();
			fail("Expected a JobSubmissionFailure.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
		}

		assertThatHABlobsHaveNotBeenRemoved();
	} finally {
		finishJob();
	}

	assertThatHABlobsHaveBeenRemoved();
}
 

@Override
public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
	log.info("Received JobGraph submission {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	try {
		if (isDuplicateJob(jobGraph.getJobID())) {
			return FutureUtils.completedExceptionally(
				new JobSubmissionException(jobGraph.getJobID(), "Job has already been submitted."));
		} else if (isPartialResourceConfigured(jobGraph)) {
			return FutureUtils.completedExceptionally(
				new JobSubmissionException(jobGraph.getJobID(), "Currently jobs is not supported if parts of the vertices have " +
						"resources configured. The limitation will be removed in future versions."));
		} else {
			return internalSubmitJob(jobGraph);
		}
	} catch (FlinkException e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 

private CompletableFuture<Acknowledge> internalSubmitJob(JobGraph jobGraph) {
	log.info("Submitting job {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	final CompletableFuture<Acknowledge> persistAndRunFuture = waitForTerminatingJobManager(jobGraph.getJobID(), jobGraph, this::persistAndRunJob)
		.thenApply(ignored -> Acknowledge.get());

	return persistAndRunFuture.handleAsync((acknowledge, throwable) -> {
		if (throwable != null) {
			cleanUpJobData(jobGraph.getJobID(), true);

			final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
			log.error("Failed to submit job {}.", jobGraph.getJobID(), strippedThrowable);
			throw new CompletionException(
				new JobSubmissionException(jobGraph.getJobID(), "Failed to submit job.", strippedThrowable));
		} else {
			return acknowledge;
		}
	}, getRpcService().getExecutor());
}
 

/**
 * Tests that a duplicate job submission won't delete any job meta data
 * (submitted job graphs, blobs, etc.).
 */
@Test
public void testDuplicateJobSubmissionDoesNotDeleteJobMetaData() throws Exception {
	submitJob();

	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		try {
			submissionFuture.get();
			fail("Expected a JobSubmissionFailure.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
		}

		assertThatHABlobsHaveNotBeenRemoved();
	} finally {
		finishJob();
	}

	assertThatHABlobsHaveBeenRemoved();
}
 

@Override
public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
	log.info("Received JobGraph submission {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	try {
		if (isDuplicateJob(jobGraph.getJobID())) {
			return FutureUtils.completedExceptionally(
				new DuplicateJobSubmissionException(jobGraph.getJobID()));
		} else if (isPartialResourceConfigured(jobGraph)) {
			return FutureUtils.completedExceptionally(
				new JobSubmissionException(jobGraph.getJobID(), "Currently jobs is not supported if parts of the vertices have " +
						"resources configured. The limitation will be removed in future versions."));
		} else {
			return internalSubmitJob(jobGraph);
		}
	} catch (FlinkException e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 

private CompletableFuture<Acknowledge> internalSubmitJob(JobGraph jobGraph) {
	log.info("Submitting job {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	final CompletableFuture<Acknowledge> persistAndRunFuture = waitForTerminatingJobManager(jobGraph.getJobID(), jobGraph, this::persistAndRunJob)
		.thenApply(ignored -> Acknowledge.get());

	return persistAndRunFuture.handleAsync((acknowledge, throwable) -> {
		if (throwable != null) {
			cleanUpJobData(jobGraph.getJobID(), true);

			final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
			log.error("Failed to submit job {}.", jobGraph.getJobID(), strippedThrowable);
			throw new CompletionException(
				new JobSubmissionException(jobGraph.getJobID(), "Failed to submit job.", strippedThrowable));
		} else {
			return acknowledge;
		}
	}, getRpcService().getExecutor());
}
 

@Override
public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
	log.info("Received JobGraph submission {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	try {
		if (isDuplicateJob(jobGraph.getJobID())) {
			return FutureUtils.completedExceptionally(
				new JobSubmissionException(jobGraph.getJobID(), "Job has already been submitted."));
		} else {
			return internalSubmitJob(jobGraph);
		}
	} catch (FlinkException e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 

/**
 * Tests that the uploaded blobs are being cleaned up in case of a job submission failure.
 */
@Test
public void testBlobServerCleanupWhenJobSubmissionFails() throws Exception {
	failJobMasterCreationWith.set(() -> new FlinkException("Test exception."));
	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		submissionFuture.get();
		fail("Job submission was expected to fail.");
	} catch (ExecutionException ee) {
		assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
	}

	assertThatHABlobsHaveBeenRemoved();
}
 

/**
 * Tests that the uploaded blobs are being cleaned up in case of a job submission failure.
 */
@Test
public void testBlobServerCleanupWhenJobSubmissionFails() throws Exception {
	failJobMasterCreationWith.set(() -> new FlinkException("Test exception."));
	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		submissionFuture.get();
		fail("Job submission was expected to fail.");
	} catch (ExecutionException ee) {
		assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
	}

	assertThatHABlobsHaveBeenRemoved();
}
 

/**
 * Tests that we can submit a job to the Dispatcher which then spawns a
 * new JobManagerRunner.
 */
@Test
public void testJobSubmissionWithPartialResourceConfigured() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	CompletableFuture<UUID> leaderFuture = dispatcherLeaderElectionService.isLeader(UUID.randomUUID());

	// wait for the leader to be elected
	leaderFuture.get();

	DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	ResourceSpec resourceSpec = ResourceSpec.newBuilder().setCpuCores(2).build();

	final JobVertex firstVertex = new JobVertex("firstVertex");
	firstVertex.setInvokableClass(NoOpInvokable.class);
	firstVertex.setResources(resourceSpec, resourceSpec);

	final JobVertex secondVertex = new JobVertex("secondVertex");
	secondVertex.setInvokableClass(NoOpInvokable.class);

	JobGraph jobGraphWithTwoVertices = new JobGraph(TEST_JOB_ID, "twoVerticesJob", firstVertex, secondVertex);
	jobGraphWithTwoVertices.setAllowQueuedScheduling(true);

	CompletableFuture<Acknowledge> acknowledgeFuture = dispatcherGateway.submitJob(jobGraphWithTwoVertices, TIMEOUT);

	try {
		acknowledgeFuture.get();
		fail("job submission should have failed");
	}
	catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent());
	}
}
 

/**
 * Tests that the uploaded blobs are being cleaned up in case of a job submission failure.
 */
@Test
public void testBlobServerCleanupWhenJobSubmissionFails() throws Exception {
	startDispatcher(new FailingJobManagerRunnerFactory(new FlinkException("Test exception")));
	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		submissionFuture.get();
		fail("Job submission was expected to fail.");
	} catch (ExecutionException ee) {
		assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
	}

	assertThatHABlobsHaveBeenRemoved();
}
 

/**
 * Tests that we can submit a job to the Dispatcher which then spawns a
 * new JobManagerRunner.
 */
@Test
public void testJobSubmissionWithPartialResourceConfigured() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	ResourceSpec resourceSpec = ResourceSpec.newBuilder(2.0, 0).build();

	final JobVertex firstVertex = new JobVertex("firstVertex");
	firstVertex.setInvokableClass(NoOpInvokable.class);
	firstVertex.setResources(resourceSpec, resourceSpec);

	final JobVertex secondVertex = new JobVertex("secondVertex");
	secondVertex.setInvokableClass(NoOpInvokable.class);

	JobGraph jobGraphWithTwoVertices = new JobGraph(TEST_JOB_ID, "twoVerticesJob", firstVertex, secondVertex);

	CompletableFuture<Acknowledge> acknowledgeFuture = dispatcherGateway.submitJob(jobGraphWithTwoVertices, TIMEOUT);

	try {
		acknowledgeFuture.get();
		fail("job submission should have failed");
	}
	catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent());
	}
}
 

@Test
public void onAddedJobGraph_failingRecoveredJobSubmission_failsFatally() throws Exception {
	final TestingDispatcherGateway dispatcherGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> FutureUtils.completedExceptionally(new JobSubmissionException(jobGraph.getJobID(), "test exception")))
		.build();

	runOnAddedJobGraphTest(dispatcherGateway, this::verifyOnAddedJobGraphResultFailsFatally);
}
 

private void verifyOnAddedJobGraphResultFailsFatally(TestingFatalErrorHandler fatalErrorHandler) {
		final Throwable actualCause = fatalErrorHandler.getErrorFuture().join();

		assertTrue(ExceptionUtils.findThrowable(actualCause, JobSubmissionException.class).isPresent());

		fatalErrorHandler.clearError();
}