org.apache.flink.runtime.webmonitor.TestingDispatcherGateway Java Examples

@Test
public void testDispatcherIsCancelledWhenOneJobIsCancelled() throws Exception {
	final CompletableFuture<ApplicationStatus> clusterShutdownStatus = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.CANCELED))
			.setClusterShutdownFunction((status) -> {
				clusterShutdownStatus.complete(status);
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createCancelledJobResult(jobId)));

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	// wait until the bootstrap "thinks" it's done, also makes sure that we don't
	// fail the future exceptionally with a JobCancelledException
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(clusterShutdownStatus.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(ApplicationStatus.CANCELED));
}
 

@Test
public void testSerializationFailureHandling() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance()), mockGateway);
		Assert.fail();
	} catch (RestHandlerException rhe) {
		Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
	}
}
 

@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 

private void runOnAddedJobGraphTest(TestingDispatcherGateway dispatcherGateway, ThrowingConsumer<TestingFatalErrorHandler, Exception> verificationLogic) throws Exception {
	jobGraphStore = TestingJobGraphStore.newBuilder()
		.setInitialJobGraphs(Collections.singleton(JOB_GRAPH))
		.build();
	dispatcherServiceFactory = TestingDispatcherServiceFactory.newBuilder()
		.setCreateFunction((dispatcherId, jobGraphs, jobGraphWriter) -> {
			assertThat(jobGraphs, containsInAnyOrder(JOB_GRAPH));

			return TestingDispatcherGatewayService.newBuilder()
				.setDispatcherGateway(dispatcherGateway)
				.build();
		})
		.build();

	try (final SessionDispatcherLeaderProcess dispatcherLeaderProcess = createDispatcherLeaderProcess()) {
		dispatcherLeaderProcess.start();

		dispatcherLeaderProcess.getDispatcherGateway().get();

		dispatcherLeaderProcess.onAddedJobGraph(JOB_GRAPH.getJobID());

		verificationLogic.accept(fatalErrorHandler);
	}
}
 

@BeforeClass
public static void setup() throws Exception {
	init();
	final GatewayRetriever<TestingDispatcherGateway> gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
	final Time timeout = Time.seconds(10);
	final Map<String, String> responseHeaders = Collections.emptyMap();
	final Executor executor = TestingUtils.defaultExecutor();

	handler = new JarRunHandler(
		gatewayRetriever,
		timeout,
		responseHeaders,
		JarRunHeaders.getInstance(),
		jarDir,
		new Configuration(),
		executor);
}
 

@Test
public void testJobIdDefaultsToZeroWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 0L)));
}
 

@Test
public void testStaticJobId() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 2L)));
}
 

@Test
public void testStaticJobIdWithHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 2L)));
}
 

@BeforeClass
public static void setup() throws Exception {
	init();
	final GatewayRetriever<TestingDispatcherGateway> gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
	final Time timeout = Time.seconds(10);
	final Map<String, String> responseHeaders = Collections.emptyMap();
	final Executor executor = TestingUtils.defaultExecutor();

	handler = new JarRunHandler(
		gatewayRetriever,
		timeout,
		responseHeaders,
		JarRunHeaders.getInstance(),
		jarDir,
		new Configuration(),
		executor,
		ConfigurationVerifyingDetachedApplicationRunner::new);
}
 

@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 

@Test
public void testSerializationFailureHandling() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance()), mockGateway);
		Assert.fail();
	} catch (RestHandlerException rhe) {
		Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
	}
}
 

@Test
public void testApplicationIsStoppedWhenStoppingBootstrap() throws Exception {
	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.RUNNING));

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	ScheduledFuture<?> applicationExecutionFuture = bootstrap.getApplicationExecutionFuture();

	bootstrap.stop();

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// verify that the application task is being cancelled
	assertThat(applicationExecutionFuture.isCancelled(), is(true));
}
 

@Test
public void testClusterShutdownWhenApplicationFails() throws Exception {
	// we're "listening" on this to be completed to verify that the cluster
	// is being shut down from the ApplicationDispatcherBootstrap
	final CompletableFuture<ApplicationStatus> externalShutdownFuture = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FAILED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createFailedJobResult(jobId)))
			.setClusterShutdownFunction((status) -> {
				externalShutdownFuture.complete(status);
				return CompletableFuture.completedFuture(Acknowledge.get());
			});

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// verify that the dispatcher is actually being shut down
	assertThat(externalShutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(ApplicationStatus.FAILED));
}
 

@Test
public void testSerializationFailureHandling() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance()), mockGateway);
		Assert.fail();
	} catch (RestHandlerException rhe) {
		Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
	}
}
 

@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 

@Test
public void testClusterShutdownWhenApplicationSucceeds() throws Exception {
	// we're "listening" on this to be completed to verify that the cluster
	// is being shut down from the ApplicationDispatcherBootstrap
	final CompletableFuture<ApplicationStatus> externalShutdownFuture = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)))
			.setClusterShutdownFunction((status) -> {
				externalShutdownFuture.complete(status);
				return CompletableFuture.completedFuture(Acknowledge.get());
			});

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// verify that the dispatcher is actually being shut down
	assertThat(externalShutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(ApplicationStatus.SUCCEEDED));
}
 

@Test
public void testClusterShutdownWhenSubmissionFails() throws Exception {
	// we're "listening" on this to be completed to verify that the cluster
	// is being shut down from the ApplicationDispatcherBootstrap
	final CompletableFuture<ApplicationStatus> externalShutdownFuture = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				throw new FlinkRuntimeException("Nope!");
			})
			.setClusterShutdownFunction((status) -> {
				externalShutdownFuture.complete(status);
				return CompletableFuture.completedFuture(Acknowledge.get());
			});

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// verify that the dispatcher is actually being shut down
	assertThat(externalShutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(ApplicationStatus.FAILED));
}
 

@BeforeClass
public static void setup() throws Exception {
	init();
	final GatewayRetriever<TestingDispatcherGateway> gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
	final Time timeout = Time.seconds(10);
	final Map<String, String> responseHeaders = Collections.emptyMap();
	final Executor executor = TestingUtils.defaultExecutor();

	handler = new JarRunHandler(
		gatewayRetriever,
		timeout,
		responseHeaders,
		JarRunHeaders.getInstance(),
		jarDir,
		new Configuration(),
		executor);
}
 

@Test
public void testApplicationTaskFinishesWhenApplicationFinishes() throws Exception {
	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	ScheduledFuture<?> applicationExecutionFuture = bootstrap.getApplicationExecutionFuture();

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// make sure the task finishes
	applicationExecutionFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
 

@Test
public void testApplicationFailsAsSoonAsOneJobFails() throws Throwable {
	final ConcurrentLinkedDeque<JobID> submittedJobIds = new ConcurrentLinkedDeque<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobIds.add(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> {
				// we only fail one of the jobs, the first one, the others will "keep" running
				// indefinitely
				if (jobId.equals(submittedJobIds.peek())) {
					return CompletableFuture.completedFuture(JobStatus.FAILED);
				}
				// never finish the other jobs
				return CompletableFuture.completedFuture(JobStatus.RUNNING);
			})
			.setRequestJobResultFunction(jobId -> {
				// we only fail one of the jobs, the first one, the other will "keep" running
				// indefinitely. If we didn't have this the test would hang forever.
				if (jobId.equals(submittedJobIds.peek())) {
					return CompletableFuture.completedFuture(createFailedJobResult(jobId));
				}
				// never finish the other jobs
				return new CompletableFuture<>();
			});

	final CompletableFuture<Void> applicationFuture = runApplication(dispatcherBuilder, 2);

	assertException(applicationFuture, JobExecutionException.class);
}
 

@Test
public void testApplicationSucceedsWhenAllJobsSucceed() throws Exception {
	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture = runApplication(dispatcherBuilder, 3);

	// this would block indefinitely if the applications don't finish
	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
 

@Test
public void onAddedJobGraph_duplicateJobSubmissionDueToFalsePositive_willBeIgnored() throws Exception {
	final TestingDispatcherGateway dispatcherGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> FutureUtils.completedExceptionally(new DuplicateJobSubmissionException(jobGraph.getJobID())))
		.build();

	runOnAddedJobGraphTest(dispatcherGateway, this::verifyOnAddedJobGraphResultDidNotFail);
}
 

@Test
public void testExceptionThrownWhenApplicationContainsNoJobs() throws Throwable {
	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, 0);

	assertException(applicationFuture, ApplicationExecutionException.class);
}
 

@Test
public void testFailedJobSubmission() throws Exception {
	final String errorMessage = "test";
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobgraph -> FutureUtils.completedExceptionally(new Exception(errorMessage)))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();

	JobGraph jobGraph = new JobGraph("testjob");
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(jobGraph);
	}
	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(
				request,
				EmptyMessageParameters.getInstance(),
				Collections.emptyMap(),
				Collections.emptyMap(),
				Collections.singletonList(jobGraphFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		Throwable t = ExceptionUtils.stripExecutionException(e);
		Assert.assertEquals(errorMessage, t.getMessage());
	}
}
 

@Test
public void testRejectionOnCountMismatch() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}
	final Path countExceedingFile = TEMPORARY_FOLDER.newFile().toPath();

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(jobGraphFile.toFile(), countExceedingFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		ExceptionUtils.findThrowable(e, candidate -> candidate instanceof RestHandlerException && candidate.getMessage().contains("count"));
	}
}
 

@Test
public void testClusterShutdownWhenStoppingBootstrap() throws Exception {
	// we're "listening" on this to be completed to verify that the cluster
	// is being shut down from the ApplicationDispatcherBootstrap
	final CompletableFuture<ApplicationStatus> externalShutdownFuture = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.RUNNING))
			.setClusterShutdownFunction((status) -> {
				externalShutdownFuture.complete(status);
				return CompletableFuture.completedFuture(Acknowledge.get());
			});

	ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(2);

	final CompletableFuture<Acknowledge> shutdownFuture =
			bootstrap.runApplicationAndShutdownClusterAsync(dispatcherBuilder.build(), scheduledExecutor);

	bootstrap.stop();

	// wait until the bootstrap "thinks" it's done
	shutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	// verify that the dispatcher is actually being shut down
	assertThat(externalShutdownFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(ApplicationStatus.UNKNOWN));
}
 

private CompletableFuture<Void> runApplication(
		final Configuration configuration,
		final int noOfJobs) throws Throwable {

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	return runApplication(dispatcherBuilder, configuration, noOfJobs);
}
 

private CompletableFuture<Void> runApplication(
		TestingDispatcherGateway.Builder dispatcherBuilder,
		Configuration configuration,
		int noOfJobs) throws FlinkException {

	final PackagedProgram program = getProgram(noOfJobs);

	final ApplicationDispatcherBootstrap bootstrap =
			new ApplicationDispatcherBootstrap(
					program, Collections.emptyList(), configuration);

	return bootstrap.fixJobIdAndRunApplicationAsync(
			dispatcherBuilder.build(),
			scheduledExecutor);
}
 

@Test
public void testFailedJobSubmission() throws Exception {
	final String errorMessage = "test";
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobgraph -> FutureUtils.completedExceptionally(new Exception(errorMessage)))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();

	JobGraph jobGraph = new JobGraph("testjob");
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(jobGraph);
	}
	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(
				request,
				EmptyMessageParameters.getInstance(),
				Collections.emptyMap(),
				Collections.emptyMap(),
				Collections.singletonList(jobGraphFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		Throwable t = ExceptionUtils.stripExecutionException(e);
		Assert.assertEquals(errorMessage, t.getMessage());
	}
}
 

@Test
public void testJarSubmission() throws Exception {
	final TestingDispatcherGateway restfulGateway = new TestingDispatcherGateway.Builder()
		.setBlobServerPort(blobServerResource.getBlobServerPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();
	final JarHandlers handlers = new JarHandlers(temporaryFolder.newFolder().toPath(), restfulGateway);
	final JarUploadHandler uploadHandler = handlers.uploadHandler;
	final JarListHandler listHandler = handlers.listHandler;
	final JarPlanHandler planHandler = handlers.planHandler;
	final JarRunHandler runHandler = handlers.runHandler;
	final JarDeleteHandler deleteHandler = handlers.deleteHandler;

	// targetDir property is set via surefire configuration
	final Path originalJar = Paths.get(System.getProperty("targetDir")).resolve("test-program.jar");
	final Path jar = Files.copy(originalJar, temporaryFolder.getRoot().toPath().resolve("test-program.jar"));

	final String storedJarPath = uploadJar(uploadHandler, jar, restfulGateway);
	final String storedJarName = Paths.get(storedJarPath).getFileName().toString();

	final JarListInfo postUploadListResponse = listJars(listHandler, restfulGateway);
	Assert.assertEquals(1, postUploadListResponse.jarFileList.size());
	final JarListInfo.JarFileInfo listEntry = postUploadListResponse.jarFileList.iterator().next();
	Assert.assertEquals(jar.getFileName().toString(), listEntry.name);
	Assert.assertEquals(storedJarName, listEntry.id);

	final JobPlanInfo planResponse = showPlan(planHandler, storedJarName, restfulGateway);
	// we're only interested in the core functionality so checking for a small detail is sufficient
	Assert.assertThat(planResponse.getJsonPlan(), containsString("TestProgram.java:29"));

	runJar(runHandler, storedJarName, restfulGateway);

	deleteJar(deleteHandler, storedJarName, restfulGateway);

	final JarListInfo postDeleteListResponse = listJars(listHandler, restfulGateway);
	Assert.assertEquals(0, postDeleteListResponse.jarFileList.size());
}