org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder Java Examples

/**
 * Tests that the {@link RunningJobsRegistry} entries are cleared after the
 * job reached a terminal state.
 */
@Test
public void testRunningJobsRegistryCleanup() throws Exception {
	submitJob();

	runningJobsRegistry.setJobRunning(jobId);
	assertThat(runningJobsRegistry.contains(jobId), is(true));

	resultFuture.complete(new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).setJobID(jobId).build());
	terminationFuture.complete(null);

	// wait for the clearing
	clearedJobLatch.await();

	assertThat(runningJobsRegistry.contains(jobId), is(false));
}
 

@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		// the failure cause in the execution graph should not be the cause of the canceled job result
		assertThat(expected.getCause(), is(nullValue()));
	}
}
 

/**
 * Tests that we can put {@link ArchivedExecutionGraph} into the
 * {@link FileArchivedExecutionGraphStore} and that the graph is persisted.
 */
@Test
public void testPut() throws IOException {
	final ArchivedExecutionGraph dummyExecutionGraph = new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build();
	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {

		final File storageDirectory = executionGraphStore.getStorageDir();

		// check that the storage directory is empty
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0));

		executionGraphStore.put(dummyExecutionGraph);

		// check that we have persisted the given execution graph
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1));

		assertThat(executionGraphStore.get(dummyExecutionGraph.getJobID()), new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph));
	}
}
 

@Test
public void testHABlobsAreNotRemovedIfHAJobGraphRemovalFails() throws Exception {
	submittedJobGraphStore.setRemovalFailure(new Exception("Failed to Remove future"));
	submitJob();

	ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobId)
		.setState(JobStatus.CANCELED)
		.build();

	resultFuture.complete(executionGraph);
	terminationFuture.complete(null);

	assertThat(cleanupJobFuture.get(), equalTo(jobId));
	assertThat(deleteAllHABlobsFuture.isDone(), is(false));
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

/**
 * Tests that the {@link RunningJobsRegistry} entries are cleared after the
 * job reached a terminal state.
 */
@Test
public void testRunningJobsRegistryCleanup() throws Exception {
	submitJob();

	runningJobsRegistry.setJobRunning(jobId);
	assertThat(runningJobsRegistry.contains(jobId), is(true));

	resultFuture.complete(new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).setJobID(jobId).build());
	terminationFuture.complete(null);

	// wait for the clearing
	clearedJobLatch.await();

	assertThat(runningJobsRegistry.contains(jobId), is(false));
}
 

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 

@BeforeClass
public static void setupClass() throws IOException {
	jobGraph = new JobGraph();

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();

	rpcService = new TestingRpcService();
	configuration = new Configuration();

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 

/**
 * Tests that the {@link RunningJobsRegistry} entries are cleared after the
 * job reached a terminal state.
 */
@Test
public void testRunningJobsRegistryCleanup() throws Exception {
	final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob();

	runningJobsRegistry.setJobRunning(jobId);
	assertThat(runningJobsRegistry.contains(jobId), is(true));

	final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner();
	testingJobManagerRunner.completeResultFuture(new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).setJobID(jobId).build());

	// wait for the clearing
	clearedJobLatch.await();

	assertThat(runningJobsRegistry.contains(jobId), is(false));
}
 

/**
 * Test that {@link JobResult} is cached when the job finishes.
 */
@Test
public void testCacheJobExecutionResult() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	final JobID failedJobId = new JobID();

	final JobStatus expectedState = JobStatus.FAILED;
	final ArchivedExecutionGraph failedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(failedJobId)
		.setState(expectedState)
		.setFailureCause(new ErrorInfo(new RuntimeException("expected"), 1L))
		.build();

	dispatcher.completeJobExecution(failedExecutionGraph);

	assertThat(
		dispatcherGateway.requestJobStatus(failedJobId, TIMEOUT).get(),
		equalTo(expectedState));
	assertThat(
		dispatcherGateway.requestJob(failedJobId, TIMEOUT).get(),
		equalTo(failedExecutionGraph));
}
 

@Test
public void testHABlobsAreNotRemovedIfHAJobGraphRemovalFails() throws Exception {
	jobGraphWriter = TestingJobGraphStore.newBuilder()
		.setRemoveJobGraphConsumer(
			ignored -> {
				throw new Exception("Failed to Remove future");
			})
		.withAutomaticStart()
		.build();

	final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob();

	ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobId)
		.setState(JobStatus.CANCELED)
		.build();

	final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner();
	testingJobManagerRunner.completeResultFuture(executionGraph);

	assertThat(cleanupJobFuture.get(), equalTo(jobId));
	assertThat(deleteAllHABlobsFuture.isDone(), is(false));
}
 

/**
 * Tests that we can put {@link ArchivedExecutionGraph} into the
 * {@link FileArchivedExecutionGraphStore} and that the graph is persisted.
 */
@Test
public void testPut() throws IOException {
	final ArchivedExecutionGraph dummyExecutionGraph = new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build();
	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {

		final File storageDirectory = executionGraphStore.getStorageDir();

		// check that the storage directory is empty
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0));

		executionGraphStore.put(dummyExecutionGraph);

		// check that we have persisted the given execution graph
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1));

		assertThat(executionGraphStore.get(dummyExecutionGraph.getJobID()), new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph));
	}
}
 

/**
 * Tests that we can put {@link ArchivedExecutionGraph} into the
 * {@link FileArchivedExecutionGraphStore} and that the graph is persisted.
 */
@Test
public void testPut() throws IOException {
	final ArchivedExecutionGraph dummyExecutionGraph = new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build();
	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {

		final File storageDirectory = executionGraphStore.getStorageDir();

		// check that the storage directory is empty
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0));

		executionGraphStore.put(dummyExecutionGraph);

		// check that we have persisted the given execution graph
		assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1));

		assertThat(executionGraphStore.get(dummyExecutionGraph.getJobID()), new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph));
	}
}
 

@Test
public void testHABlobsAreNotRemovedIfHAJobGraphRemovalFails() throws Exception {
	submittedJobGraphStore.setRemovalFailure(new Exception("Failed to Remove future"));
	submitJob();

	ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobId)
		.setState(JobStatus.CANCELED)
		.build();

	resultFuture.complete(executionGraph);
	terminationFuture.complete(null);

	assertThat(cleanupJobFuture.get(), equalTo(jobId));
	assertThat(deleteAllHABlobsFuture.isDone(), is(false));
}
 

@Test
public void testCompletedResult() throws Exception {
	final JobStatus jobStatus = JobStatus.FINISHED;
	final ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(TEST_JOB_ID)
		.setState(jobStatus)
		.build();

	final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder()
		.setRequestJobStatusFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(jobStatus);
			})
		.setRequestJobResultFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(JobResult.createFrom(executionGraph));
			}
		)
		.build();

	final JobExecutionResultResponseBody responseBody = jobExecutionResultHandler.handleRequest(
		testRequest,
		testingRestfulGateway).get();

	assertThat(
		responseBody.getStatus().getId(),
		equalTo(QueueStatus.Id.COMPLETED));
	assertThat(responseBody.getJobExecutionResult(), not(nullValue()));
}
 

@BeforeClass
public static void setupClass() {
	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

@Test
public void testHABlobsAreRemovedIfHAJobGraphRemovalSucceeds() throws Exception {
	final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob();

	ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobId)
		.setState(JobStatus.CANCELED)
		.build();

	final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner();
	testingJobManagerRunner.completeResultFuture(executionGraph);

	assertThat(cleanupJobFuture.get(), equalTo(jobId));
	assertThat(deleteAllHABlobsFuture.get(), equalTo(jobId));
}
 

@Test
public void testFailedJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(new FlinkException("Test exception"), 42L))
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

@Test
public void testCancelledJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

private Collection<ArchivedExecutionGraph> generateTerminalExecutionGraphs(int number) {
	final Collection<ArchivedExecutionGraph> executionGraphs = new ArrayList<>(number);

	for (int i = 0; i < number; i++) {
		final JobStatus state = GLOBALLY_TERMINAL_JOB_STATUS.get(ThreadLocalRandom.current().nextInt(GLOBALLY_TERMINAL_JOB_STATUS.size()));
		executionGraphs.add(
			new ArchivedExecutionGraphBuilder()
				.setState(state)
				.build());
	}

	return executionGraphs;
}
 

/**
 * Test that {@link JobResult} is cached when the job finishes.
 */
@Test
public void testCacheJobExecutionResult() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	final JobID failedJobId = new JobID();

	final JobStatus expectedState = JobStatus.FAILED;
	final ArchivedExecutionGraph failedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(failedJobId)
		.setState(expectedState)
		.setFailureCause(new ErrorInfo(new RuntimeException("expected"), 1L))
		.build();

	dispatcher.completeJobExecution(failedExecutionGraph);

	assertThat(
		dispatcherGateway.requestJobStatus(failedJobId, TIMEOUT).get(),
		equalTo(expectedState));
	assertThat(
		dispatcherGateway.requestJob(failedJobId, TIMEOUT).get(),
		equalTo(failedExecutionGraph));
}
 

@Test
public void testCancelledJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

private Collection<ArchivedExecutionGraph> generateTerminalExecutionGraphs(int number) {
	final Collection<ArchivedExecutionGraph> executionGraphs = new ArrayList<>(number);

	for (int i = 0; i < number; i++) {
		final JobStatus state = GLOBALLY_TERMINAL_JOB_STATUS.get(ThreadLocalRandom.current().nextInt(GLOBALLY_TERMINAL_JOB_STATUS.size()));
		executionGraphs.add(
			new ArchivedExecutionGraphBuilder()
				.setState(state)
				.build());
	}

	return executionGraphs;
}