org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus Java Examples

The following examples show how to use org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FsNegativeRunningJobsRegistryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSetFinishedAndRunning() throws Exception {
	final Path workDir = new Path(hdfsRootPath, "änother_wörk_directörü");
	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir);

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir);
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
Example #2
Source File: FsNegativeRunningJobsRegistryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSetFinishedAndRunning() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
Example #3
Source File: FsNegativeRunningJobsRegistryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCreateAndSetFinished() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// initially, without any call, the job is pending
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));

	// after set running, the job is running
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));
}
Example #4
Source File: FsNegativeRunningJobsRegistryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSetFinishedAndRunning() throws Exception {
	final Path workDir = new Path(hdfsRootPath, "änother_wörk_directörü");
	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir);

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir);
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
Example #5
Source File: FsNegativeRunningJobsRegistryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCreateAndSetFinished() throws Exception {
	final Path workDir = new Path(hdfsRootPath, "test-work-dir");
	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir);

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir);

	// initially, without any call, the job is pending
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));

	// after set running, the job is running
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));
}
Example #6
Source File: FsNegativeRunningJobsRegistryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSetFinishedAndRunning() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
Example #7
Source File: FsNegativeRunningJobsRegistryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCreateAndSetFinished() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// initially, without any call, the job is pending
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));

	// after set running, the job is running
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));
}
Example #8
Source File: FsNegativeRunningJobsRegistryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCreateAndSetFinished() throws Exception {
	final Path workDir = new Path(hdfsRootPath, "test-work-dir");
	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir);

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir);

	// initially, without any call, the job is pending
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));

	// after set running, the job is running
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));
}
Example #9
Source File: ZooKeeperRegistryTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
Example #10
Source File: JobManagerRunner.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<JobSchedulingStatus> getJobSchedulingStatus() {
	try {
		JobSchedulingStatus jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobGraph.getJobID());
		return CompletableFuture.completedFuture(jobSchedulingStatus);
	} catch (IOException e) {
		return FutureUtils.completedExceptionally(
			new FlinkException(
				String.format("Could not retrieve the job scheduling status for job %s.", jobGraph.getJobID()),
				e));
	}
}
Example #11
Source File: JobManagerRunner.java    From flink with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<Void> verifyJobSchedulingStatusAndStartJobManager(UUID leaderSessionId) {
	final CompletableFuture<JobSchedulingStatus> jobSchedulingStatusFuture = getJobSchedulingStatus();

	return jobSchedulingStatusFuture.thenCompose(
		jobSchedulingStatus -> {
			if (jobSchedulingStatus == JobSchedulingStatus.DONE) {
				return jobAlreadyDone();
			} else {
				return startJobMaster(leaderSessionId);
			}
		});
}
Example #12
Source File: JobManagerRunner.java    From flink with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<JobSchedulingStatus> getJobSchedulingStatus() {
	try {
		JobSchedulingStatus jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobGraph.getJobID());
		return CompletableFuture.completedFuture(jobSchedulingStatus);
	} catch (IOException e) {
		return FutureUtils.completedExceptionally(
			new FlinkException(
				String.format("Could not retrieve the job scheduling status for job %s.", jobGraph.getJobID()),
				e));
	}
}
Example #13
Source File: JobManagerRunner.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<Void> verifyJobSchedulingStatusAndStartJobManager(UUID leaderSessionId) {
	final CompletableFuture<JobSchedulingStatus> jobSchedulingStatusFuture = getJobSchedulingStatus();

	return jobSchedulingStatusFuture.thenCompose(
		jobSchedulingStatus -> {
			if (jobSchedulingStatus == JobSchedulingStatus.DONE) {
				return jobAlreadyDone();
			} else {
				return startJobMaster(leaderSessionId);
			}
		});
}
Example #14
Source File: ZooKeeperRegistryTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
Example #15
Source File: JobManagerRunnerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<Void> verifyJobSchedulingStatusAndStartJobManager(UUID leaderSessionId) {
	final CompletableFuture<JobSchedulingStatus> jobSchedulingStatusFuture = getJobSchedulingStatus();

	return jobSchedulingStatusFuture.thenCompose(
		jobSchedulingStatus -> {
			if (jobSchedulingStatus == JobSchedulingStatus.DONE) {
				return jobAlreadyDone();
			} else {
				return startJobMaster(leaderSessionId);
			}
		});
}
Example #16
Source File: JobManagerRunnerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
private CompletableFuture<JobSchedulingStatus> getJobSchedulingStatus() {
	try {
		JobSchedulingStatus jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobGraph.getJobID());
		return CompletableFuture.completedFuture(jobSchedulingStatus);
	} catch (IOException e) {
		return FutureUtils.completedExceptionally(
			new FlinkException(
				String.format("Could not retrieve the job scheduling status for job %s.", jobGraph.getJobID()),
				e));
	}
}
Example #17
Source File: ZooKeeperRegistryTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}